From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- .../scripts/lint/update_buildconfig_from_gradle.py | 165 +++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100755 taskcluster/scripts/lint/update_buildconfig_from_gradle.py (limited to 'taskcluster/scripts/lint/update_buildconfig_from_gradle.py') diff --git a/taskcluster/scripts/lint/update_buildconfig_from_gradle.py b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py new file mode 100755 index 0000000000..148fa19aa4 --- /dev/null +++ b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import json +import logging +import os +import re +import subprocess +import sys +from collections import defaultdict + +import yaml +from mergedeep import merge + +logger = logging.getLogger(__name__) + +_DEFAULT_GRADLE_COMMAND = ("./gradlew", "--console=plain", "--no-parallel") +_LOCAL_DEPENDENCY_PATTERN = re.compile( + r"(\+|\\)--- project :(?P\S+)\s?.*" +) + + +def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config): + project_dependencies = defaultdict(set) + gradle_projects = _get_gradle_projects(gradle_root, existing_build_config) + + logger.info(f"Looking for dependencies in {gradle_root}") + + # This is eventually going to fail if there's ever enough projects to make the + # command line too long. If that happens, we'll need to split this list up and + # run gradle more than once. + cmd = list(_DEFAULT_GRADLE_COMMAND) + cmd.extend( + [f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects)] + ) + + # Parsing output like this is not ideal but bhearsum couldn't find a way + # to get the dependencies printed in a better format. If we could convince + # gradle to spit out JSON that would be much better. + # This is filed as https://bugzilla.mozilla.org/show_bug.cgi?id=1795152 + current_project_name = None + print(f"Running command: {' '.join(cmd)}") + try: + output = subprocess.check_output(cmd, universal_newlines=True, cwd=gradle_root) + except subprocess.CalledProcessError as cpe: + print(cpe.output) + raise + for line in output.splitlines(): + # If we find the start of a new component section, update our tracking + # variable + if line.startswith("Project"): + current_project_name = line.split(":")[1].strip("'") + + # If we find a new local dependency, add it. + local_dep_match = _LOCAL_DEPENDENCY_PATTERN.search(line) + if local_dep_match: + local_dependency_name = local_dep_match.group("local_dependency_name") + if ( + local_dependency_name != current_project_name + # These lint rules are not part of android-components + and local_dependency_name != "mozilla-lint-rules" + ): + project_dependencies[current_project_name].add(local_dependency_name) + + return { + project_name: sorted(project_dependencies[project_name]) + for project_name in gradle_projects + } + + +def _get_gradle_projects(gradle_root, existing_build_config): + if gradle_root.endswith("android-components"): + return list(existing_build_config["projects"].keys()) + elif gradle_root.endswith("focus-android"): + return ["app"] + elif gradle_root.endswith("fenix"): + return ["app"] + + raise NotImplementedError(f"Cannot find gradle projects for {gradle_root}") + + +def is_dir(string): + if os.path.isdir(string): + return string + else: + raise argparse.ArgumentTypeError(f'"{string}" is not a directory') + + +def _parse_args(cmdln_args): + parser = argparse.ArgumentParser( + description="Calls gradle and generate json file with dependencies" + ) + parser.add_argument( + "gradle_root", + metavar="GRADLE_ROOT", + type=is_dir, + help="The directory where to call gradle from", + ) + return parser.parse_args(args=cmdln_args) + + +def _set_logging_config(): + logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s" + ) + + +def _merge_build_config( + existing_build_config, upstream_deps_per_project, variants_config +): + updated_build_config = { + "projects": { + project: {"upstream_dependencies": deps} + for project, deps in upstream_deps_per_project.items() + } + } + updated_variant_config = {"variants": variants_config} if variants_config else {} + return merge(existing_build_config, updated_build_config, updated_variant_config) + + +def _get_variants(gradle_root): + cmd = list(_DEFAULT_GRADLE_COMMAND) + ["printVariants"] + output_lines = subprocess.check_output( + cmd, universal_newlines=True, cwd=gradle_root + ).splitlines() + variants_line = [line for line in output_lines if line.startswith("variants: ")][0] + variants_json = variants_line.split(" ", 1)[1] + return json.loads(variants_json) + + +def _should_print_variants(gradle_root): + return gradle_root.endswith("fenix") or gradle_root.endswith("focus-android") + + +def main(): + args = _parse_args(sys.argv[1:]) + gradle_root = args.gradle_root + build_config_file = os.path.join(gradle_root, ".buildconfig.yml") + _set_logging_config() + + with open(build_config_file) as f: + existing_build_config = yaml.safe_load(f) + + upstream_deps_per_project = _get_upstream_deps_per_gradle_project( + gradle_root, existing_build_config + ) + + variants_config = ( + _get_variants(gradle_root) if _should_print_variants(gradle_root) else {} + ) + merged_build_config = _merge_build_config( + existing_build_config, upstream_deps_per_project, variants_config + ) + + with open(build_config_file, "w") as f: + yaml.safe_dump(merged_build_config, f) + logger.info(f"Updated {build_config_file} with latest gradle config!") + + +__name__ == "__main__" and main() -- cgit v1.2.3