diff options
Diffstat (limited to 'fluent-bit/lib/librdkafka-2.1.0/packaging/nuget/packaging.py')
| -rwxr-xr-x | fluent-bit/lib/librdkafka-2.1.0/packaging/nuget/packaging.py | 448 |
1 files changed, 448 insertions, 0 deletions
diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/nuget/packaging.py b/fluent-bit/lib/librdkafka-2.1.0/packaging/nuget/packaging.py new file mode 100755 index 000000000..c4dab806d --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/nuget/packaging.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +# +# Packaging script. +# Assembles packages using CI artifacts. +# + +import sys +import re +import os +import shutil +from fnmatch import fnmatch +from string import Template +from zfile import zfile +import boto3 +import magic + +if sys.version_info[0] < 3: + from urllib import unquote as _unquote +else: + from urllib.parse import unquote as _unquote + + +def unquote(path): + # Removes URL escapes, and normalizes the path by removing ./. + path = _unquote(path) + if path[:2] == './': + return path[2:] + return path + + +# Rename token values +rename_vals = {'plat': {'windows': 'win'}, + 'arch': {'x86_64': 'x64', + 'amd64': 'x64', + 'i386': 'x86', + 'win32': 'x86'}} + +# Filemagic arch mapping. +# key is (plat, arch, file_extension), value is a compiled filemagic regex. +# This is used to verify that an artifact has the expected file type. +magic_patterns = { + ('win', 'x64', '.dll'): re.compile('PE32.*DLL.* x86-64, for MS Windows'), + ('win', 'x86', '.dll'): + re.compile('PE32.*DLL.* Intel 80386, for MS Windows'), + ('win', 'x64', '.lib'): re.compile('current ar archive'), + ('win', 'x86', '.lib'): re.compile('current ar archive'), + ('linux', 'x64', '.so'): re.compile('ELF 64.* x86-64'), + ('linux', 'arm64', '.so'): re.compile('ELF 64.* ARM aarch64'), + ('osx', 'x64', '.dylib'): re.compile('Mach-O 64.* x86_64'), + ('osx', 'arm64', '.dylib'): re.compile('Mach-O 64.*arm64')} + +magic = magic.Magic() + + +def magic_mismatch(path, a): + """ Verify that the filemagic for \\p path matches for artifact \\p a. + Returns True if the magic file info does NOT match. + Returns False if no matching is needed or the magic matches. """ + k = (a.info.get('plat', None), a.info.get('arch', None), + os.path.splitext(path)[1]) + pattern = magic_patterns.get(k, None) + if pattern is None: + return False + + minfo = magic.id_filename(path) + if not pattern.match(minfo): + print( + f"Warning: {path} magic \"{minfo}\" " + f"does not match expected {pattern} for key {k}") + return True + + return False + + +# Collects CI artifacts from S3 storage, downloading them +# to a local directory, or collecting already downloaded artifacts from +# local directory. +# +# The artifacts' folder in the S3 bucket must have the following token +# format: +# <token>-[<value>]__ (repeat) +# +# Recognized tokens (unrecognized tokens are ignored): +# p - project (e.g., "confluent-kafka-python") +# bld - builder (e.g., "travis") +# plat - platform ("osx", "linux", ..) +# dist - distro or runtime ("centos6", "mingw", "msvcr", "alpine", ..). +# arch - arch ("x64", ..) +# tag - git tag +# sha - git sha +# bid - builder's build-id +# bldtype - Release, Debug (appveyor) +# lnk - Linkage ("std", "static", "all" (both std and static)) +# extra - Extra build options, typically "gssapi" (for cyrus-sasl linking). + +# +# Example: +# librdkafka/p-librdkafka__bld-travis__plat-linux__arch-x64__tag-v0.0.62__sha-d051b2c19eb0c118991cd8bc5cf86d8e5e446cde__bid-1562.1/librdkafka.tar.gz + + +class MissingArtifactError(Exception): + pass + + +s3_bucket = 'librdkafka-ci-packages' +dry_run = False + + +class Artifact (object): + def __init__(self, arts, path, info=None): + self.path = path + # Remove unexpanded AppVeyor $(..) tokens from filename + self.fname = re.sub(r'\$\([^\)]+\)', '', os.path.basename(path)) + slpath = os.path.join(os.path.dirname(path), self.fname) + if os.path.isfile(slpath): + # Already points to local file in correct location + self.lpath = slpath + else: + # Prepare download location in dlpath + self.lpath = os.path.join(arts.dlpath, slpath) + + if info is None: + self.info = dict() + else: + # Assign the map and convert all keys to lower case + self.info = {k.lower(): v for k, v in info.items()} + # Rename values, e.g., 'plat':'windows' to 'plat':'win' + for k, v in self.info.items(): + rdict = rename_vals.get(k, None) + if rdict is not None: + self.info[k] = rdict.get(v, v) + + # Score value for sorting + self.score = 0 + + # AppVeyor symbol builds are of less value + if self.fname.find('.symbols.') != -1: + self.score -= 10 + + self.arts = arts + arts.artifacts.append(self) + + def __repr__(self): + return self.path + + def __lt__(self, other): + return self.score < other.score + + def download(self): + """ Download artifact from S3 and store in local directory .lpath. + If the artifact is already downloaded nothing is done. """ + if os.path.isfile(self.lpath) and os.path.getsize(self.lpath) > 0: + return + print('Downloading %s' % self.path) + if dry_run: + return + ldir = os.path.dirname(self.lpath) + if not os.path.isdir(ldir): + os.makedirs(ldir, 0o755) + self.arts.s3_bucket.download_file(self.path, self.lpath) + + +class Artifacts (object): + def __init__(self, match, dlpath): + super(Artifacts, self).__init__() + self.match = match + self.artifacts = list() + # Download directory (make sure it ends with a path separator) + if not dlpath.endswith(os.path.sep): + dlpath = os.path.join(dlpath, '') + self.dlpath = dlpath + if not os.path.isdir(self.dlpath): + if not dry_run: + os.makedirs(self.dlpath, 0o755) + + def collect_single(self, path, req_tag=True): + """ Collect single artifact, be it in S3 or locally. + :param: path string: S3 or local (relative) path + :param: req_tag bool: Require tag to match. + """ + + # For local files, strip download path. + # Also ignore any parent directories. + if path.startswith(self.dlpath): + folder = os.path.basename(os.path.dirname(path[len(self.dlpath):])) + else: + folder = os.path.basename(os.path.dirname(path)) + + # The folder contains the tokens needed to perform + # matching of project, gitref, etc. + rinfo = re.findall(r'(?P<tag>[^-]+)-(?P<val>.*?)(?:__|$)', folder) + if rinfo is None or len(rinfo) == 0: + print('Incorrect folder/file name format for %s' % folder) + return None + + info = dict(rinfo) + + # Ignore AppVeyor Debug builds + if info.get('bldtype', '').lower() == 'debug': + print('Ignoring debug artifact %s' % folder) + return None + + tag = info.get('tag', None) + if tag is not None and (len(tag) == 0 or tag.startswith('$(')): + # AppVeyor doesn't substite $(APPVEYOR_REPO_TAG_NAME) + # with an empty value when not set, it leaves that token + # in the string - so translate that to no tag. + del info['tag'] + + # Perform matching + unmatched = list() + for m, v in self.match.items(): + if m not in info or info[m] != v: + unmatched.append(f"{m} = {v}") + + # Make sure all matches were satisfied, unless this is a + # common artifact. + if info.get('p', '') != 'common' and len(unmatched) > 0: + return None + + return Artifact(self, path, info) + + def collect_s3(self): + """ Collect and download build-artifacts from S3 based on + git reference """ + print( + 'Collecting artifacts matching %s from S3 bucket %s' % + (self.match, s3_bucket)) + self.s3 = boto3.resource('s3') + self.s3_bucket = self.s3.Bucket(s3_bucket) + self.s3_client = boto3.client('s3') + + # note: list_objects will return at most 1000 objects per call, + # use continuation token to read full list. + cont_token = None + more = True + while more: + if cont_token is not None: + res = self.s3_client.list_objects_v2( + Bucket=s3_bucket, + Prefix='librdkafka/', + ContinuationToken=cont_token) + else: + res = self.s3_client.list_objects_v2(Bucket=s3_bucket, + Prefix='librdkafka/') + + if res.get('IsTruncated') is True: + cont_token = res.get('NextContinuationToken') + else: + more = False + + for item in res.get('Contents'): + self.collect_single(item.get('Key')) + + for a in self.artifacts: + a.download() + + def collect_local(self, path, req_tag=True): + """ Collect artifacts from a local directory possibly previously + collected from s3 """ + for f in [os.path.join(dp, f) for dp, dn, + filenames in os.walk(path) for f in filenames]: + if not os.path.isfile(f): + continue + self.collect_single(f, req_tag) + + +class Mapping (object): + """ Maps/matches a file in an input release artifact to + the output location of the package, based on attributes and paths. """ + + def __init__(self, attributes, artifact_fname_glob, path_in_artifact, + output_pkg_path=None, artifact_fname_excludes=[]): + """ + @param attributes A dict of artifact attributes that must match. + If an attribute name (dict key) is prefixed + with "!" (e.g., "!plat") then the attribute + must not match. + @param artifact_fname_glob Match artifacts with this filename glob. + @param path_in_artifact On match, extract this file in the artifact,.. + @param output_pkg_path ..and write it to this location in the package. + Defaults to path_in_artifact. + @param artifact_fname_excludes Exclude artifacts matching these + filenames. + + Pass a list of Mapping objects to FIXME to perform all mappings. + """ + super(Mapping, self).__init__() + self.attributes = attributes + self.fname_glob = artifact_fname_glob + self.input_path = path_in_artifact + if output_pkg_path is None: + self.output_path = self.input_path + else: + self.output_path = output_pkg_path + self.name = self.output_path + self.fname_excludes = artifact_fname_excludes + + def __str__(self): + return self.name + + +class Package (object): + """ Generic Package class + A Package is a working container for one or more output + packages for a specific package type (e.g., nuget) """ + + def __init__(self, version, arts): + super(Package, self).__init__() + self.version = version + self.arts = arts + # These may be overwritten by specific sub-classes: + self.artifacts = arts.artifacts + # Staging path, filled in later. + self.stpath = None + self.kv = {'version': version} + self.files = dict() + + def add_file(self, file): + self.files[file] = True + + def build(self): + """ Build package output(s), return a list of paths " + to built packages """ + raise NotImplementedError + + def cleanup(self): + """ Optional cleanup routine for removing temporary files, etc. """ + pass + + def render(self, fname, destpath='.'): + """ Render template in file fname and save to destpath/fname, + where destpath is relative to stpath """ + + outf = os.path.join(self.stpath, destpath, fname) + + if not os.path.isdir(os.path.dirname(outf)): + os.makedirs(os.path.dirname(outf), 0o0755) + + with open(os.path.join('templates', fname), 'r') as tf: + tmpl = Template(tf.read()) + with open(outf, 'w') as of: + of.write(tmpl.substitute(self.kv)) + + self.add_file(outf) + + def copy_template(self, fname, target_fname=None, destpath='.'): + """ Copy template file to destpath/fname + where destpath is relative to stpath """ + + if target_fname is None: + target_fname = fname + outf = os.path.join(self.stpath, destpath, target_fname) + + if not os.path.isdir(os.path.dirname(outf)): + os.makedirs(os.path.dirname(outf), 0o0755) + + shutil.copy(os.path.join('templates', fname), outf) + + self.add_file(outf) + + def apply_mappings(self): + """ Applies a list of Mapping to match and extract files from + matching artifacts. If any of the listed Mappings can not be + fulfilled an exception is raised. """ + + assert self.mappings + assert len(self.mappings) > 0 + + for m in self.mappings: + + artifact = None + for a in self.arts.artifacts: + found = True + + for attr in m.attributes: + if attr[0] == '!': + # Require attribute NOT to match + origattr = attr + attr = attr[1:] + + if attr in a.info and \ + a.info[attr] != m.attributes[origattr]: + found = False + break + else: + # Require attribute to match + if attr not in a.info or \ + a.info[attr] != m.attributes[attr]: + found = False + break + + if not fnmatch(a.fname, m.fname_glob): + found = False + + for exclude in m.fname_excludes: + if exclude in a.fname: + found = False + break + + if found: + artifact = a + break + + if artifact is None: + raise MissingArtifactError( + '%s: unable to find artifact with tags %s matching "%s"' % + (m, str(m.attributes), m.fname_glob)) + + output_path = os.path.join(self.stpath, m.output_path) + + try: + zfile.ZFile.extract(artifact.lpath, m.input_path, output_path) +# except KeyError: +# continue + except Exception as e: + raise Exception( + '%s: file not found in archive %s: %s. Files in archive are:\n%s' % # noqa: E501 + (m, artifact.lpath, e, '\n'.join(zfile.ZFile( + artifact.lpath).getnames()))) + + # Check that the file type matches. + if magic_mismatch(output_path, a): + os.unlink(output_path) + continue + + # All mappings found and extracted. + + def verify(self, path): + """ Verify package content based on the previously defined mappings """ + + missing = list() + with zfile.ZFile(path, 'r') as zf: + print('Verifying %s:' % path) + + # Zipfiles may url-encode filenames, unquote them before matching. + pkgd = [unquote(x) for x in zf.getnames()] + missing = [x for x in self.mappings if x.output_path not in pkgd] + + if len(missing) > 0: + print( + 'Missing files in package %s:\n%s' % + (path, '\n'.join([str(x) for x in missing]))) + print('Actual: %s' % '\n'.join(pkgd)) + return False + + print('OK - %d expected files found' % len(self.mappings)) + return True |