diff options
Diffstat (limited to 'python/mozbuild/mozbuild/vendor/moz_yaml.py')
| -rw-r--r-- | python/mozbuild/mozbuild/vendor/moz_yaml.py | 770 |
1 files changed, 770 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/vendor/moz_yaml.py b/python/mozbuild/mozbuild/vendor/moz_yaml.py new file mode 100644 index 0000000000..51210e19b2 --- /dev/null +++ b/python/mozbuild/mozbuild/vendor/moz_yaml.py @@ -0,0 +1,770 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +# Utility package for working with moz.yaml files. +# +# Requires `pyyaml` and `voluptuous` +# (both are in-tree under third_party/python) + +import errno +import os +import re + +import voluptuous +import yaml +from voluptuous import ( + All, + Boolean, + FqdnUrl, + In, + Invalid, + Length, + Match, + Msg, + Required, + Schema, + Unique, +) +from yaml.error import MarkedYAMLError + +# TODO ensure this matches the approved list of licenses +VALID_LICENSES = [ + # Standard Licenses (as per https://spdx.org/licenses/) + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "BSD-3-Clause-Clear", + "BSL-1.0", + "CC0-1.0", + "ISC", + "ICU", + "LGPL-2.1", + "LGPL-3.0", + "MIT", + "MPL-1.1", + "MPL-2.0", + "Unlicense", + "WTFPL", + "Zlib", + # Unique Licenses + "ACE", # http://www.cs.wustl.edu/~schmidt/ACE-copying.html + "Anti-Grain-Geometry", # http://www.antigrain.com/license/index.html + "JPNIC", # https://www.nic.ad.jp/ja/idn/idnkit/download/index.html + "Khronos", # https://www.khronos.org/openmaxdl + "libpng", # http://www.libpng.org/pub/png/src/libpng-LICENSE.txt + "Unicode", # http://www.unicode.org/copyright.html +] + +VALID_SOURCE_HOSTS = ["gitlab", "googlesource", "github", "angle", "codeberg"] + +""" +--- +# Third-Party Library Template +# All fields are mandatory unless otherwise noted + +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: product name + component: component name + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: name of the package + + description: short (one line) description + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: package's homepage url + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: identifier + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: sha + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: MPL-2.0 + + # If the package's license is specified in a particular file, + # this is the name of the file. + # optional + license-file: COPYING + + # If there are any mozilla-specific notes you want to put + # about a library, they can be put here. + notes: Notes about the library + +# Configuration for the automated vendoring system. +# optional +vendoring: + + # Repository URL to vendor from + # eg. https://github.com/kinetiknz/nestegg + # Any repository host can be specified here, however initially we'll only + # support automated vendoring from selected sources. + url: source url (generally repository clone url) + + # Type of hosting for the upstream repository + # Valid values are 'gitlab', 'github', googlesource + source-hosting: gitlab + + # Type of Vendoring + # This is either 'regular', 'individual-files', or 'rust' + # If omitted, will default to 'regular' + flavor: rust + + # Type of git reference (commit, tag) to track updates from. + # You cannot use tag tracking with the individual-files flavor + # If omitted, will default to tracking commits. + tracking: commit + + # Base directory of the location where the source files will live in-tree. + # If omitted, will default to the location the moz.yaml file is in. + vendor-directory: third_party/directory + + # Allows skipping certain steps of the vendoring process. + # Most useful if e.g. vendoring upstream is complicated and should be done by a script + # The valid steps that can be skipped are listed below + skip-vendoring-steps: + - fetch + - keep + - include + - exclude + - move-contents + - hg-add + - spurious-check + - update-moz-yaml + - update-moz-build + + # List of patch files to apply after vendoring. Applied in the order + # specified, and alphabetically if globbing is used. Patches must apply + # cleanly before changes are pushed. + # Patch files should be relative to the vendor-directory rather than the gecko + # root directory. + # All patch files are implicitly added to the keep file list. + # optional + patches: + - file + - path/to/file + - path/*.patch + - path/** # Captures all files and subdirectories below path + - path/* # Captures all files but _not_ subdirectories below path. Equivalent to `path/` + + # List of files that are not removed from the destination directory while vendoring + # in a new version of the library. Intended for mozilla files not present in upstream. + # Implicitly contains "moz.yaml", "moz.build", and any files referenced in + # "patches" + # optional + keep: + - file + - path/to/file + - another/path + - *.mozilla + + # Files/paths that will not be vendored from the upstream repository + # Implicitly contains ".git", and ".gitignore" + # optional + exclude: + - file + - path/to/file + - another/path + - docs + - src/*.test + + # Files/paths that will always be vendored from source repository, even if + # they would otherwise be excluded by "exclude". + # optional + include: + - file + - path/to/file + - another/path + - docs/LICENSE.* + + # Files that are modified as part of the update process. + # To avoid creating updates that don't update anything, ./mach vendor will detect + # if any in-tree files have changed. If there are files that are always changed + # during an update process (e.g. version numbers or source revisions), list them + # here to avoid having them counted as substative changes. + # This field does NOT support directories or globbing + # optional + generated: + - '{yaml_dir}/vcs_version.h' + + # If neither "exclude" or "include" are set, all files will be vendored + # Files/paths in "include" will always be vendored, even if excluded + # eg. excluding "docs/" then including "docs/LICENSE" will vendor just the + # LICENSE file from the docs directory + + # All three file/path parameters ("keep", "exclude", and "include") support + # filenames, directory names, and globs/wildcards. + + # Actions to take after updating. Applied in order. + # The action subfield is required. It must be one of: + # - copy-file + # - move-file + # - move-dir + # - replace-in-file + # - replace-in-file-regex + # - delete-path + # - run-script + # Unless otherwise noted, all subfields of action are required. + # + # If the action is copy-file, move-file, or move-dir: + # from is the source file + # to is the destination + # + # If the action is replace-in-file or replace-in-file-regex: + # pattern is what in the file to search for. It is an exact strng match. + # with is the string to replace it with. Accepts the special keyword + # '{revision}' for the commit we are updating to. + # File is the file to replace it in. + # + # If the action is delete-path + # path is the file or directory to recursively delete + # + # If the action is run-script: + # script is the script to run + # cwd is the directory the script should run with as its cwd + # args is a list of arguments to pass to the script + # + # If the action is run-command: + # command is the command to run + # Unlike run-script, `command` is _not_ processed to be relative + # to the vendor directory, and is passed directly to python's + # execution code without any path substitution or manipulation + # cwd is the directory the command should run with as its cwd + # args is a list of arguments to pass to the command + # + # + # Unless specified otherwise, all files/directories are relative to the + # vendor-directory. If the vendor-directory is different from the + # directory of the yaml file, the keyword '{yaml_dir}' may be used + # to make the path relative to that directory. + # 'run-script' supports the addictional keyword {cwd} which, if used, + # must only be used at the beginning of the path. + # + # optional + update-actions: + - action: copy-file + from: include/vcs_version.h.in + to: '{yaml_dir}/vcs_version.h' + + - action: replace-in-file + pattern: '@VCS_TAG@' + with: '{revision}' + file: '{yaml_dir}/vcs_version.h' + + - action: delete-path + path: '{yaml_dir}/config' + + - action: run-script + script: '{cwd}/generate_sources.sh' + cwd: '{yaml_dir}' + + +# Configuration for automatic updating system. +# optional +updatebot: + + # TODO: allow multiple users to be specified + # Phabricator username for a maintainer of the library, used for assigning + # reviewers. For a review group, preface with #, such as "#build"" + maintainer-phab: tjr + + # Bugzilla email address for a maintainer of the library, used for needinfos + maintainer-bz: tom@mozilla.com + + # Optional: A preset for ./mach try to use. If present, fuzzy-query and fuzzy-paths will + # be ignored. If it, fuzzy-query, and fuzzy-path are omitted, ./mach try auto will be used + try-preset: media + + # Optional: A query string for ./mach try fuzzy. If try-preset, it and fuzzy-paths are omitted + # then ./mach try auto will be used + fuzzy-query: media + + # Optional: An array of test paths for ./mach try fuzzy. If try-preset, it and fuzzy-query are + # omitted then ./mach try auto will be used + fuzzy-paths: ['media'] + + # The tasks that Updatebot can run. Only one of each task is currently permitted + # optional + tasks: + - type: commit-alert + branch: upstream-branch-name + cc: ["bugzilla@email.address", "another@example.com"] + needinfo: ["bugzilla@email.address", "another@example.com"] + enabled: True + filter: security + frequency: every + platform: windows + blocking: 1234 + - type: vendoring + branch: master + enabled: False + + # frequency can be 'every', 'release', 'N weeks', 'N commits' + # or 'N weeks, M commits' requiring satisfying both constraints. + frequency: 2 weeks +""" + +RE_SECTION = re.compile(r"^(\S[^:]*):").search +RE_FIELD = re.compile(r"^\s\s([^:]+):\s+(\S+)$").search + + +class MozYamlVerifyError(Exception): + def __init__(self, filename, error): + self.filename = filename + self.error = error + + def __str__(self): + return "%s: %s" % (self.filename, self.error) + + +def load_moz_yaml(filename, verify=True, require_license_file=True): + """Loads and verifies the specified manifest.""" + + # Load and parse YAML. + try: + with open(filename, "r") as f: + manifest = yaml.load(f, Loader=yaml.BaseLoader) + except IOError as e: + if e.errno == errno.ENOENT: + raise MozYamlVerifyError(filename, "Failed to find manifest: %s" % filename) + raise + except MarkedYAMLError as e: + raise MozYamlVerifyError(filename, e) + + if not verify: + return manifest + + # Verify schema. + if "schema" not in manifest: + raise MozYamlVerifyError(filename, 'Missing manifest "schema"') + if manifest["schema"] == "1": + schema = _schema_1() + schema_additional = _schema_1_additional + schema_transform = _schema_1_transform + else: + raise MozYamlVerifyError(filename, "Unsupported manifest schema") + + try: + schema(manifest) + schema_additional(filename, manifest, require_license_file=require_license_file) + manifest = schema_transform(manifest) + except (voluptuous.Error, ValueError) as e: + raise MozYamlVerifyError(filename, e) + + return manifest + + +def _schema_1(): + """Returns Voluptuous Schema object.""" + return Schema( + { + Required("schema"): "1", + Required("bugzilla"): { + Required("product"): All(str, Length(min=1)), + Required("component"): All(str, Length(min=1)), + }, + "origin": { + Required("name"): All(str, Length(min=1)), + Required("description"): All(str, Length(min=1)), + "notes": All(str, Length(min=1)), + Required("url"): FqdnUrl(), + Required("license"): Msg(License(), msg="Unsupported License"), + "license-file": All(str, Length(min=1)), + Required("release"): All(str, Length(min=1)), + # The following regex defines a valid git reference + # The first group [^ ~^:?*[\]] matches 0 or more times anything + # that isn't a Space, ~, ^, :, ?, *, or ] + # The second group [^ ~^:?*[\]\.]+ matches 1 or more times + # anything that isn't a Space, ~, ^, :, ?, *, [, ], or . + "revision": Match(r"^[^ ~^:?*[\]]*[^ ~^:?*[\]\.]+$"), + }, + "updatebot": { + Required("maintainer-phab"): All(str, Length(min=1)), + Required("maintainer-bz"): All(str, Length(min=1)), + "try-preset": All(str, Length(min=1)), + "fuzzy-query": All(str, Length(min=1)), + "fuzzy-paths": All([str], Length(min=1)), + "tasks": All( + UpdatebotTasks(), + [ + { + Required("type"): In( + ["vendoring", "commit-alert"], + msg="Invalid type specified in tasks", + ), + "branch": All(str, Length(min=1)), + "enabled": Boolean(), + "cc": Unique([str]), + "needinfo": Unique([str]), + "filter": In( + ["none", "security", "source-extensions"], + msg="Invalid filter value specified in tasks", + ), + "source-extensions": Unique([str]), + "blocking": Match(r"^[0-9]+$"), + "frequency": Match( + r"^(every|release|[1-9][0-9]* weeks?|[1-9][0-9]* commits?|" + + r"[1-9][0-9]* weeks?, ?[1-9][0-9]* commits?)$" + ), + "platform": Match(r"^(windows|linux)$"), + } + ], + ), + }, + "vendoring": { + Required("url"): FqdnUrl(), + Required("source-hosting"): All( + str, + Length(min=1), + In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"), + ), + "tracking": Match(r"^(commit|tag)$"), + "flavor": Match(r"^(regular|rust|individual-files)$"), + "skip-vendoring-steps": Unique([str]), + "vendor-directory": All(str, Length(min=1)), + "patches": Unique([str]), + "keep": Unique([str]), + "exclude": Unique([str]), + "include": Unique([str]), + "generated": Unique([str]), + "individual-files": [ + { + Required("upstream"): All(str, Length(min=1)), + Required("destination"): All(str, Length(min=1)), + } + ], + "individual-files-default-upstream": All(str, Length(min=1)), + "individual-files-default-destination": All(str, Length(min=1)), + "individual-files-list": Unique([str]), + "update-actions": All( + UpdateActions(), + [ + { + Required("action"): In( + [ + "copy-file", + "move-file", + "move-dir", + "replace-in-file", + "replace-in-file-regex", + "run-script", + "run-command", + "delete-path", + ], + msg="Invalid action specified in update-actions", + ), + "from": All(str, Length(min=1)), + "to": All(str, Length(min=1)), + "pattern": All(str, Length(min=1)), + "with": All(str, Length(min=1)), + "file": All(str, Length(min=1)), + "script": All(str, Length(min=1)), + "command": All(str, Length(min=1)), + "args": All([All(str, Length(min=1))]), + "cwd": All(str, Length(min=1)), + "path": All(str, Length(min=1)), + } + ], + ), + }, + } + ) + + +def _schema_1_additional(filename, manifest, require_license_file=True): + """Additional schema/validity checks""" + + vendor_directory = os.path.dirname(filename) + if "vendoring" in manifest and "vendor-directory" in manifest["vendoring"]: + vendor_directory = manifest["vendoring"]["vendor-directory"] + + # LICENSE file must exist, except for Rust crates which are exempted + # because the license is required to be specified in the Cargo.toml file + if require_license_file and "origin" in manifest: + files = [f.lower() for f in os.listdir(vendor_directory)] + if ( + not ( + "license-file" in manifest["origin"] + and manifest["origin"]["license-file"].lower() in files + ) + and not ( + "license" in files + or "license.txt" in files + or "license.rst" in files + or "license.html" in files + or "license.md" in files + ) + and not ( + "vendoring" in manifest + and manifest["vendoring"].get("flavor", "regular") == "rust" + ) + ): + license = manifest["origin"]["license"] + if isinstance(license, list): + license = "/".join(license) + raise ValueError("Failed to find %s LICENSE file" % license) + + # Cannot vendor without an origin. + if "vendoring" in manifest and "origin" not in manifest: + raise ValueError('"vendoring" requires an "origin"') + + # Cannot vendor without a computer-readable revision. + if "vendoring" in manifest and "revision" not in manifest["origin"]: + raise ValueError( + 'If "vendoring" is present, "revision" must be present in "origin"' + ) + + # The Rust and Individual Flavor type precludes a lot of options + # individual-files could, in theory, use several of these, but until we have a use case let's + # disallow them so we're not worrying about whether they work. When we need them we can make + # sure they do. + if ( + "vendoring" in manifest + and manifest["vendoring"].get("flavor", "regular") != "regular" + ): + for i in [ + "skip-vendoring-steps", + "keep", + "exclude", + "include", + "generated", + ]: + if i in manifest["vendoring"]: + raise ValueError("A non-regular flavor of update cannot use '%s'" % i) + + if manifest["vendoring"].get("flavor", "regular") == "rust": + for i in [ + "update-actions", + ]: + if i in manifest["vendoring"]: + raise ValueError("A rust flavor of update cannot use '%s'" % i) + + # Ensure that only individual-files flavor uses those options + if ( + "vendoring" in manifest + and manifest["vendoring"].get("flavor", "regular") != "individual-files" + ): + if ( + "individual-files" in manifest["vendoring"] + or "individual-files-list" in manifest["vendoring"] + ): + raise ValueError( + "Only individual-files flavor of update can use 'individual-files'" + ) + + # Ensure that the individual-files flavor has all the correct options + if ( + "vendoring" in manifest + and manifest["vendoring"].get("flavor", "regular") == "individual-files" + ): + # Because the only way we can determine the latest tag is by doing a local clone, + # we don't want to do that for individual-files flavors because those flavors are + # usually on gigantic repos we don't want to clone for such a simple thing. + if manifest["vendoring"].get("tracking", "commit") == "tag": + raise ValueError( + "You cannot use tag tracking with the individual-files flavor. (Sorry.)" + ) + + # We need either individual-files or individual-files-list + if ( + "individual-files" not in manifest["vendoring"] + and "individual-files-list" not in manifest["vendoring"] + ): + raise ValueError( + "The individual-files flavor must include either " + + "'individual-files' or 'individual-files-list'" + ) + # For whichever we have, make sure we don't have the other and we don't have + # options we shouldn't or lack ones we should. + if "individual-files" in manifest["vendoring"]: + if "individual-files-list" in manifest["vendoring"]: + raise ValueError( + "individual-files-list is mutually exclusive with individual-files" + ) + if "individual-files-default-upstream" in manifest["vendoring"]: + raise ValueError( + "individual-files-default-upstream can only be used with individual-files-list" + ) + if "individual-files-default-destination" in manifest["vendoring"]: + raise ValueError( + "individual-files-default-destination can only be used " + + "with individual-files-list" + ) + if "individual-files-list" in manifest["vendoring"]: + if "individual-files" in manifest["vendoring"]: + raise ValueError( + "individual-files is mutually exclusive with individual-files-list" + ) + if "individual-files-default-upstream" not in manifest["vendoring"]: + raise ValueError( + "individual-files-default-upstream must be used with individual-files-list" + ) + if "individual-files-default-destination" not in manifest["vendoring"]: + raise ValueError( + "individual-files-default-destination must be used with individual-files-list" + ) + + if "updatebot" in manifest: + # If there are Updatebot tasks, then certain fields must be present and + # defaults need to be set. + if "tasks" in manifest["updatebot"]: + if "vendoring" not in manifest or "url" not in manifest["vendoring"]: + raise ValueError( + "If Updatebot tasks are specified, a vendoring url must be included." + ) + + if "try-preset" in manifest["updatebot"]: + for f in ["fuzzy-query", "fuzzy-paths"]: + if f in manifest["updatebot"]: + raise ValueError( + "If 'try-preset' is specified, then %s cannot be" % f + ) + + # Check for a simple YAML file + with open(filename, "r") as f: + has_schema = False + for line in f.readlines(): + m = RE_SECTION(line) + if m: + if m.group(1) == "schema": + has_schema = True + break + if not has_schema: + raise ValueError("Not simple YAML") + + +# Do type conversion for the few things that need it. +# Everythig is parsed as a string to (a) not cause problems with revisions that +# are only numerals and (b) not strip leading zeros from the numbers if we just +# converted them to string +def _schema_1_transform(manifest): + if "updatebot" in manifest: + if "tasks" in manifest["updatebot"]: + for i in range(len(manifest["updatebot"]["tasks"])): + if "enabled" in manifest["updatebot"]["tasks"][i]: + val = manifest["updatebot"]["tasks"][i]["enabled"] + manifest["updatebot"]["tasks"][i]["enabled"] = ( + val.lower() == "true" or val.lower() == "yes" + ) + return manifest + + +class UpdateActions(object): + """Voluptuous validator which verifies the update actions(s) are valid.""" + + def __call__(self, values): + for v in values: + if "action" not in v: + raise Invalid("All file-update entries must specify a valid action") + if v["action"] in ["copy-file", "move-file", "move-dir"]: + if "from" not in v or "to" not in v or len(v.keys()) != 3: + raise Invalid( + "%s action must (only) specify 'from' and 'to' keys" + % v["action"] + ) + elif v["action"] in ["replace-in-file", "replace-in-file-regex"]: + if ( + "pattern" not in v + or "with" not in v + or "file" not in v + or len(v.keys()) != 4 + ): + raise Invalid( + "replace-in-file action must (only) specify " + + "'pattern', 'with', and 'file' keys" + ) + elif v["action"] == "delete-path": + if "path" not in v or len(v.keys()) != 2: + raise Invalid( + "delete-path action must (only) specify the 'path' key" + ) + elif v["action"] == "run-script": + if "script" not in v or "cwd" not in v: + raise Invalid( + "run-script action must specify 'script' and 'cwd' keys" + ) + if set(v.keys()) - set(["args", "cwd", "script", "action"]) != set(): + raise Invalid( + "run-script action may only specify 'script', 'cwd', and 'args' keys" + ) + elif v["action"] == "run-command": + if "command" not in v or "cwd" not in v: + raise Invalid( + "run-command action must specify 'command' and 'cwd' keys" + ) + if set(v.keys()) - set(["args", "cwd", "command", "action"]) != set(): + raise Invalid( + "run-command action may only specify 'command', 'cwd', and 'args' keys" + ) + else: + # This check occurs before the validator above, so the above is + # redundant but we leave it to be verbose. + raise Invalid("Supplied action " + v["action"] + " is invalid.") + return values + + def __repr__(self): + return "UpdateActions" + + +class UpdatebotTasks(object): + """Voluptuous validator which verifies the updatebot task(s) are valid.""" + + def __call__(self, values): + seenTaskTypes = set() + for v in values: + if "type" not in v: + raise Invalid("All updatebot tasks must specify a valid type") + + if v["type"] in seenTaskTypes: + raise Invalid("Only one type of each task is currently supported") + seenTaskTypes.add(v["type"]) + + if v["type"] == "vendoring": + for i in ["filter", "branch", "source-extensions"]: + if i in v: + raise Invalid( + "'%s' is only valid for commit-alert task types" % i + ) + elif v["type"] == "commit-alert": + pass + else: + # This check occurs before the validator above, so the above is + # redundant but we leave it to be verbose. + raise Invalid("Supplied type " + v["type"] + " is invalid.") + return values + + def __repr__(self): + return "UpdatebotTasks" + + +class License(object): + """Voluptuous validator which verifies the license(s) are valid as per our + allow list.""" + + def __call__(self, values): + if isinstance(values, str): + values = [values] + elif not isinstance(values, list): + raise Invalid("Must be string or list") + for v in values: + if v not in VALID_LICENSES: + raise Invalid("Bad License") + return values + + def __repr__(self): + return "License" |