Merging upstream version 126.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 290 insertions, 0 deletions

diff --git a/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py
new file mode 100755
index 0000000000..2df9cc044f
--- /dev/null
+++ b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import subprocess
+import sys
+
+from update_buildconfig_from_gradle import main as update_build_config
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+PROJECT_DIR = os.path.realpath(os.path.join(CURRENT_DIR, "..", "..", ".."))
+OUTPUT_DIR = os.path.join(PROJECT_DIR, "artifacts")
+BUILDCONFIG_DIFF_FILE_NAME = "buildconfig.diff"
+BUILDCONFIG_DIFF_FILE = os.path.join(OUTPUT_DIR, BUILDCONFIG_DIFF_FILE_NAME)
+BUILDCONFIG_FILE_NAME = ".buildconfig.yml"
+
+logger = logging.getLogger(__name__)
+
+
+def _buildconfig_files_diff():
+    cmd = [
+        "hg",
+        "diff",
+        "--rev",
+        "draft() and ancestors(.)",
+        "-I",
+        "**/.buildconfig.yml",
+    ]
+    p = subprocess.run(cmd, capture_output=True, universal_newlines=True)
+    return p.stdout
+
+
+def _execute_taskcluster_steps(diff, task_id):
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+    with open(BUILDCONFIG_DIFF_FILE, mode="w") as f:
+        f.write(diff)
+    tc_root_url = os.environ["TASKCLUSTER_ROOT_URL"]
+    artifact_url = f"{tc_root_url}/api/queue/v1/task/{task_id}/artifacts/public%2F{BUILDCONFIG_DIFF_FILE_NAME}"  # noqa E501
+    message = f"""{BUILDCONFIG_FILE_NAME} file changed! Please update it by running:
+
+curl --location --compressed {artifact_url} | git apply
+
+Then commit and push!
+"""
+    logger.error(message)
+
+
+def _execute_local_steps():
+    logger.error(f"{BUILDCONFIG_FILE_NAME} file updated! Please commit these changes.")
+
+
+def main():
+    update_build_config()
+    diff = _buildconfig_files_diff()
+    if diff:
+        task_id = os.environ.get("TASK_ID")
+        if task_id:
+            _execute_taskcluster_steps(diff, task_id)
+        else:
+            _execute_local_steps()
+        sys.exit(1)
+
+    logger.info(f"All good! {BUILDCONFIG_FILE_NAME} is up-to-date with gradle.")
+
+
+__name__ == "__main__" and main()
diff --git a/taskcluster/scripts/lint/requirements.in b/taskcluster/scripts/lint/requirements.in
new file mode 100644
index 0000000000..d5f4369f9e
--- /dev/null
+++ b/taskcluster/scripts/lint/requirements.in
@@ -0,0 +1,2 @@
+mergedeep
+pyyaml
diff --git a/taskcluster/scripts/lint/requirements.txt b/taskcluster/scripts/lint/requirements.txt
new file mode 100644
index 0000000000..1a3ce6f714
--- /dev/null
+++ b/taskcluster/scripts/lint/requirements.txt
@@ -0,0 +1,52 @@
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
+#
+mergedeep==1.3.4 \
+    --hash=sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8 \
+    --hash=sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307
+    # via -r requirements.in
+pyyaml==6.0 \
+    --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \
+    --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
+    --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
+    --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \
+    --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \
+    --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \
+    --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \
+    --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \
+    --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \
+    --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \
+    --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \
+    --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \
+    --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \
+    --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \
+    --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \
+    --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \
+    --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \
+    --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \
+    --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \
+    --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \
+    --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \
+    --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \
+    --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \
+    --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \
+    --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \
+    --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \
+    --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \
+    --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \
+    --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \
+    --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \
+    --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \
+    --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \
+    --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \
+    --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \
+    --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \
+    --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \
+    --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \
+    --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \
+    --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
+    --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
+    # via -r requirements.in
diff --git a/taskcluster/scripts/lint/update_buildconfig_from_gradle.py b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py
new file mode 100755
index 0000000000..148fa19aa4
--- /dev/null
+++ b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+from collections import defaultdict
+
+import yaml
+from mergedeep import merge
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_GRADLE_COMMAND = ("./gradlew", "--console=plain", "--no-parallel")
+_LOCAL_DEPENDENCY_PATTERN = re.compile(
+    r"(\+|\\)--- project :(?P<local_dependency_name>\S+)\s?.*"
+)
+
+
+def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config):
+    project_dependencies = defaultdict(set)
+    gradle_projects = _get_gradle_projects(gradle_root, existing_build_config)
+
+    logger.info(f"Looking for dependencies in {gradle_root}")
+
+    # This is eventually going to fail if there's ever enough projects to make the
+    # command line too long. If that happens, we'll need to split this list up and
+    # run gradle more than once.
+    cmd = list(_DEFAULT_GRADLE_COMMAND)
+    cmd.extend(
+        [f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects)]
+    )
+
+    # Parsing output like this is not ideal but bhearsum couldn't find a way
+    # to get the dependencies printed in a better format. If we could convince
+    # gradle to spit out JSON that would be much better.
+    # This is filed as https://bugzilla.mozilla.org/show_bug.cgi?id=1795152
+    current_project_name = None
+    print(f"Running command: {' '.join(cmd)}")
+    try:
+        output = subprocess.check_output(cmd, universal_newlines=True, cwd=gradle_root)
+    except subprocess.CalledProcessError as cpe:
+        print(cpe.output)
+        raise
+    for line in output.splitlines():
+        # If we find the start of a new component section, update our tracking
+        # variable
+        if line.startswith("Project"):
+            current_project_name = line.split(":")[1].strip("'")
+
+        # If we find a new local dependency, add it.
+        local_dep_match = _LOCAL_DEPENDENCY_PATTERN.search(line)
+        if local_dep_match:
+            local_dependency_name = local_dep_match.group("local_dependency_name")
+            if (
+                local_dependency_name != current_project_name
+                # These lint rules are not part of android-components
+                and local_dependency_name != "mozilla-lint-rules"
+            ):
+                project_dependencies[current_project_name].add(local_dependency_name)
+
+    return {
+        project_name: sorted(project_dependencies[project_name])
+        for project_name in gradle_projects
+    }
+
+
+def _get_gradle_projects(gradle_root, existing_build_config):
+    if gradle_root.endswith("android-components"):
+        return list(existing_build_config["projects"].keys())
+    elif gradle_root.endswith("focus-android"):
+        return ["app"]
+    elif gradle_root.endswith("fenix"):
+        return ["app"]
+
+    raise NotImplementedError(f"Cannot find gradle projects for {gradle_root}")
+
+
+def is_dir(string):
+    if os.path.isdir(string):
+        return string
+    else:
+        raise argparse.ArgumentTypeError(f'"{string}" is not a directory')
+
+
+def _parse_args(cmdln_args):
+    parser = argparse.ArgumentParser(
+        description="Calls gradle and generate json file with dependencies"
+    )
+    parser.add_argument(
+        "gradle_root",
+        metavar="GRADLE_ROOT",
+        type=is_dir,
+        help="The directory where to call gradle from",
+    )
+    return parser.parse_args(args=cmdln_args)
+
+
+def _set_logging_config():
+    logging.basicConfig(
+        level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s"
+    )
+
+
+def _merge_build_config(
+    existing_build_config, upstream_deps_per_project, variants_config
+):
+    updated_build_config = {
+        "projects": {
+            project: {"upstream_dependencies": deps}
+            for project, deps in upstream_deps_per_project.items()
+        }
+    }
+    updated_variant_config = {"variants": variants_config} if variants_config else {}
+    return merge(existing_build_config, updated_build_config, updated_variant_config)
+
+
+def _get_variants(gradle_root):
+    cmd = list(_DEFAULT_GRADLE_COMMAND) + ["printVariants"]
+    output_lines = subprocess.check_output(
+        cmd, universal_newlines=True, cwd=gradle_root
+    ).splitlines()
+    variants_line = [line for line in output_lines if line.startswith("variants: ")][0]
+    variants_json = variants_line.split(" ", 1)[1]
+    return json.loads(variants_json)
+
+
+def _should_print_variants(gradle_root):
+    return gradle_root.endswith("fenix") or gradle_root.endswith("focus-android")
+
+
+def main():
+    args = _parse_args(sys.argv[1:])
+    gradle_root = args.gradle_root
+    build_config_file = os.path.join(gradle_root, ".buildconfig.yml")
+    _set_logging_config()
+
+    with open(build_config_file) as f:
+        existing_build_config = yaml.safe_load(f)
+
+    upstream_deps_per_project = _get_upstream_deps_per_gradle_project(
+        gradle_root, existing_build_config
+    )
+
+    variants_config = (
+        _get_variants(gradle_root) if _should_print_variants(gradle_root) else {}
+    )
+    merged_build_config = _merge_build_config(
+        existing_build_config, upstream_deps_per_project, variants_config
+    )
+
+    with open(build_config_file, "w") as f:
+        yaml.safe_dump(merged_build_config, f)
+    logger.info(f"Updated {build_config_file} with latest gradle config!")
+
+
+__name__ == "__main__" and main()