From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- .../scripts/lint/is_buildconfig_yml_up_to_date.py | 71 +++++++++ taskcluster/scripts/lint/requirements.in | 2 + taskcluster/scripts/lint/requirements.txt | 52 +++++++ .../scripts/lint/update_buildconfig_from_gradle.py | 165 +++++++++++++++++++++ 4 files changed, 290 insertions(+) create mode 100755 taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py create mode 100644 taskcluster/scripts/lint/requirements.in create mode 100644 taskcluster/scripts/lint/requirements.txt create mode 100755 taskcluster/scripts/lint/update_buildconfig_from_gradle.py (limited to 'taskcluster/scripts/lint') diff --git a/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py new file mode 100755 index 0000000000..2df9cc044f --- /dev/null +++ b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import os +import subprocess +import sys + +from update_buildconfig_from_gradle import main as update_build_config + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.realpath(os.path.join(CURRENT_DIR, "..", "..", "..")) +OUTPUT_DIR = os.path.join(PROJECT_DIR, "artifacts") +BUILDCONFIG_DIFF_FILE_NAME = "buildconfig.diff" +BUILDCONFIG_DIFF_FILE = os.path.join(OUTPUT_DIR, BUILDCONFIG_DIFF_FILE_NAME) +BUILDCONFIG_FILE_NAME = ".buildconfig.yml" + +logger = logging.getLogger(__name__) + + +def _buildconfig_files_diff(): + cmd = [ + "hg", + "diff", + "--rev", + "draft() and ancestors(.)", + "-I", + "**/.buildconfig.yml", + ] + p = subprocess.run(cmd, capture_output=True, universal_newlines=True) + return p.stdout + + +def _execute_taskcluster_steps(diff, task_id): + os.makedirs(OUTPUT_DIR, exist_ok=True) + with open(BUILDCONFIG_DIFF_FILE, mode="w") as f: + f.write(diff) + tc_root_url = os.environ["TASKCLUSTER_ROOT_URL"] + artifact_url = f"{tc_root_url}/api/queue/v1/task/{task_id}/artifacts/public%2F{BUILDCONFIG_DIFF_FILE_NAME}" # noqa E501 + message = f"""{BUILDCONFIG_FILE_NAME} file changed! Please update it by running: + +curl --location --compressed {artifact_url} | git apply + +Then commit and push! +""" + logger.error(message) + + +def _execute_local_steps(): + logger.error(f"{BUILDCONFIG_FILE_NAME} file updated! Please commit these changes.") + + +def main(): + update_build_config() + diff = _buildconfig_files_diff() + if diff: + task_id = os.environ.get("TASK_ID") + if task_id: + _execute_taskcluster_steps(diff, task_id) + else: + _execute_local_steps() + sys.exit(1) + + logger.info(f"All good! {BUILDCONFIG_FILE_NAME} is up-to-date with gradle.") + + +__name__ == "__main__" and main() diff --git a/taskcluster/scripts/lint/requirements.in b/taskcluster/scripts/lint/requirements.in new file mode 100644 index 0000000000..d5f4369f9e --- /dev/null +++ b/taskcluster/scripts/lint/requirements.in @@ -0,0 +1,2 @@ +mergedeep +pyyaml diff --git a/taskcluster/scripts/lint/requirements.txt b/taskcluster/scripts/lint/requirements.txt new file mode 100644 index 0000000000..1a3ce6f714 --- /dev/null +++ b/taskcluster/scripts/lint/requirements.txt @@ -0,0 +1,52 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements.txt requirements.in +# +mergedeep==1.3.4 \ + --hash=sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8 \ + --hash=sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307 + # via -r requirements.in +pyyaml==6.0 \ + --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via -r requirements.in diff --git a/taskcluster/scripts/lint/update_buildconfig_from_gradle.py b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py new file mode 100755 index 0000000000..148fa19aa4 --- /dev/null +++ b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import json +import logging +import os +import re +import subprocess +import sys +from collections import defaultdict + +import yaml +from mergedeep import merge + +logger = logging.getLogger(__name__) + +_DEFAULT_GRADLE_COMMAND = ("./gradlew", "--console=plain", "--no-parallel") +_LOCAL_DEPENDENCY_PATTERN = re.compile( + r"(\+|\\)--- project :(?P\S+)\s?.*" +) + + +def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config): + project_dependencies = defaultdict(set) + gradle_projects = _get_gradle_projects(gradle_root, existing_build_config) + + logger.info(f"Looking for dependencies in {gradle_root}") + + # This is eventually going to fail if there's ever enough projects to make the + # command line too long. If that happens, we'll need to split this list up and + # run gradle more than once. + cmd = list(_DEFAULT_GRADLE_COMMAND) + cmd.extend( + [f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects)] + ) + + # Parsing output like this is not ideal but bhearsum couldn't find a way + # to get the dependencies printed in a better format. If we could convince + # gradle to spit out JSON that would be much better. + # This is filed as https://bugzilla.mozilla.org/show_bug.cgi?id=1795152 + current_project_name = None + print(f"Running command: {' '.join(cmd)}") + try: + output = subprocess.check_output(cmd, universal_newlines=True, cwd=gradle_root) + except subprocess.CalledProcessError as cpe: + print(cpe.output) + raise + for line in output.splitlines(): + # If we find the start of a new component section, update our tracking + # variable + if line.startswith("Project"): + current_project_name = line.split(":")[1].strip("'") + + # If we find a new local dependency, add it. + local_dep_match = _LOCAL_DEPENDENCY_PATTERN.search(line) + if local_dep_match: + local_dependency_name = local_dep_match.group("local_dependency_name") + if ( + local_dependency_name != current_project_name + # These lint rules are not part of android-components + and local_dependency_name != "mozilla-lint-rules" + ): + project_dependencies[current_project_name].add(local_dependency_name) + + return { + project_name: sorted(project_dependencies[project_name]) + for project_name in gradle_projects + } + + +def _get_gradle_projects(gradle_root, existing_build_config): + if gradle_root.endswith("android-components"): + return list(existing_build_config["projects"].keys()) + elif gradle_root.endswith("focus-android"): + return ["app"] + elif gradle_root.endswith("fenix"): + return ["app"] + + raise NotImplementedError(f"Cannot find gradle projects for {gradle_root}") + + +def is_dir(string): + if os.path.isdir(string): + return string + else: + raise argparse.ArgumentTypeError(f'"{string}" is not a directory') + + +def _parse_args(cmdln_args): + parser = argparse.ArgumentParser( + description="Calls gradle and generate json file with dependencies" + ) + parser.add_argument( + "gradle_root", + metavar="GRADLE_ROOT", + type=is_dir, + help="The directory where to call gradle from", + ) + return parser.parse_args(args=cmdln_args) + + +def _set_logging_config(): + logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s" + ) + + +def _merge_build_config( + existing_build_config, upstream_deps_per_project, variants_config +): + updated_build_config = { + "projects": { + project: {"upstream_dependencies": deps} + for project, deps in upstream_deps_per_project.items() + } + } + updated_variant_config = {"variants": variants_config} if variants_config else {} + return merge(existing_build_config, updated_build_config, updated_variant_config) + + +def _get_variants(gradle_root): + cmd = list(_DEFAULT_GRADLE_COMMAND) + ["printVariants"] + output_lines = subprocess.check_output( + cmd, universal_newlines=True, cwd=gradle_root + ).splitlines() + variants_line = [line for line in output_lines if line.startswith("variants: ")][0] + variants_json = variants_line.split(" ", 1)[1] + return json.loads(variants_json) + + +def _should_print_variants(gradle_root): + return gradle_root.endswith("fenix") or gradle_root.endswith("focus-android") + + +def main(): + args = _parse_args(sys.argv[1:]) + gradle_root = args.gradle_root + build_config_file = os.path.join(gradle_root, ".buildconfig.yml") + _set_logging_config() + + with open(build_config_file) as f: + existing_build_config = yaml.safe_load(f) + + upstream_deps_per_project = _get_upstream_deps_per_gradle_project( + gradle_root, existing_build_config + ) + + variants_config = ( + _get_variants(gradle_root) if _should_print_variants(gradle_root) else {} + ) + merged_build_config = _merge_build_config( + existing_build_config, upstream_deps_per_project, variants_config + ) + + with open(build_config_file, "w") as f: + yaml.safe_dump(merged_build_config, f) + logger.info(f"Updated {build_config_file} with latest gradle config!") + + +__name__ == "__main__" and main() -- cgit v1.2.3