summaryrefslogtreecommitdiffstats
path: root/taskcluster/scripts/lint
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/scripts/lint')
-rwxr-xr-xtaskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py71
-rw-r--r--taskcluster/scripts/lint/requirements.in2
-rw-r--r--taskcluster/scripts/lint/requirements.txt52
-rwxr-xr-xtaskcluster/scripts/lint/update_buildconfig_from_gradle.py165
4 files changed, 290 insertions, 0 deletions
diff --git a/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py
new file mode 100755
index 0000000000..2df9cc044f
--- /dev/null
+++ b/taskcluster/scripts/lint/is_buildconfig_yml_up_to_date.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import subprocess
+import sys
+
+from update_buildconfig_from_gradle import main as update_build_config
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+PROJECT_DIR = os.path.realpath(os.path.join(CURRENT_DIR, "..", "..", ".."))
+OUTPUT_DIR = os.path.join(PROJECT_DIR, "artifacts")
+BUILDCONFIG_DIFF_FILE_NAME = "buildconfig.diff"
+BUILDCONFIG_DIFF_FILE = os.path.join(OUTPUT_DIR, BUILDCONFIG_DIFF_FILE_NAME)
+BUILDCONFIG_FILE_NAME = ".buildconfig.yml"
+
+logger = logging.getLogger(__name__)
+
+
+def _buildconfig_files_diff():
+ cmd = [
+ "hg",
+ "diff",
+ "--rev",
+ "draft() and ancestors(.)",
+ "-I",
+ "**/.buildconfig.yml",
+ ]
+ p = subprocess.run(cmd, capture_output=True, universal_newlines=True)
+ return p.stdout
+
+
+def _execute_taskcluster_steps(diff, task_id):
+ os.makedirs(OUTPUT_DIR, exist_ok=True)
+ with open(BUILDCONFIG_DIFF_FILE, mode="w") as f:
+ f.write(diff)
+ tc_root_url = os.environ["TASKCLUSTER_ROOT_URL"]
+ artifact_url = f"{tc_root_url}/api/queue/v1/task/{task_id}/artifacts/public%2F{BUILDCONFIG_DIFF_FILE_NAME}" # noqa E501
+ message = f"""{BUILDCONFIG_FILE_NAME} file changed! Please update it by running:
+
+curl --location --compressed {artifact_url} | git apply
+
+Then commit and push!
+"""
+ logger.error(message)
+
+
+def _execute_local_steps():
+ logger.error(f"{BUILDCONFIG_FILE_NAME} file updated! Please commit these changes.")
+
+
+def main():
+ update_build_config()
+ diff = _buildconfig_files_diff()
+ if diff:
+ task_id = os.environ.get("TASK_ID")
+ if task_id:
+ _execute_taskcluster_steps(diff, task_id)
+ else:
+ _execute_local_steps()
+ sys.exit(1)
+
+ logger.info(f"All good! {BUILDCONFIG_FILE_NAME} is up-to-date with gradle.")
+
+
+__name__ == "__main__" and main()
diff --git a/taskcluster/scripts/lint/requirements.in b/taskcluster/scripts/lint/requirements.in
new file mode 100644
index 0000000000..d5f4369f9e
--- /dev/null
+++ b/taskcluster/scripts/lint/requirements.in
@@ -0,0 +1,2 @@
+mergedeep
+pyyaml
diff --git a/taskcluster/scripts/lint/requirements.txt b/taskcluster/scripts/lint/requirements.txt
new file mode 100644
index 0000000000..1a3ce6f714
--- /dev/null
+++ b/taskcluster/scripts/lint/requirements.txt
@@ -0,0 +1,52 @@
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+# pip-compile --generate-hashes --output-file=requirements.txt requirements.in
+#
+mergedeep==1.3.4 \
+ --hash=sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8 \
+ --hash=sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307
+ # via -r requirements.in
+pyyaml==6.0 \
+ --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \
+ --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
+ --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
+ --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \
+ --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \
+ --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \
+ --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \
+ --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \
+ --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \
+ --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \
+ --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \
+ --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \
+ --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \
+ --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \
+ --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \
+ --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \
+ --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \
+ --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \
+ --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \
+ --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \
+ --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \
+ --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \
+ --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \
+ --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \
+ --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \
+ --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \
+ --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \
+ --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \
+ --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \
+ --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \
+ --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \
+ --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \
+ --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \
+ --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \
+ --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \
+ --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \
+ --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \
+ --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \
+ --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
+ --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
+ # via -r requirements.in
diff --git a/taskcluster/scripts/lint/update_buildconfig_from_gradle.py b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py
new file mode 100755
index 0000000000..148fa19aa4
--- /dev/null
+++ b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+from collections import defaultdict
+
+import yaml
+from mergedeep import merge
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_GRADLE_COMMAND = ("./gradlew", "--console=plain", "--no-parallel")
+_LOCAL_DEPENDENCY_PATTERN = re.compile(
+ r"(\+|\\)--- project :(?P<local_dependency_name>\S+)\s?.*"
+)
+
+
+def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config):
+ project_dependencies = defaultdict(set)
+ gradle_projects = _get_gradle_projects(gradle_root, existing_build_config)
+
+ logger.info(f"Looking for dependencies in {gradle_root}")
+
+ # This is eventually going to fail if there's ever enough projects to make the
+ # command line too long. If that happens, we'll need to split this list up and
+ # run gradle more than once.
+ cmd = list(_DEFAULT_GRADLE_COMMAND)
+ cmd.extend(
+ [f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects)]
+ )
+
+ # Parsing output like this is not ideal but bhearsum couldn't find a way
+ # to get the dependencies printed in a better format. If we could convince
+ # gradle to spit out JSON that would be much better.
+ # This is filed as https://bugzilla.mozilla.org/show_bug.cgi?id=1795152
+ current_project_name = None
+ print(f"Running command: {' '.join(cmd)}")
+ try:
+ output = subprocess.check_output(cmd, universal_newlines=True, cwd=gradle_root)
+ except subprocess.CalledProcessError as cpe:
+ print(cpe.output)
+ raise
+ for line in output.splitlines():
+ # If we find the start of a new component section, update our tracking
+ # variable
+ if line.startswith("Project"):
+ current_project_name = line.split(":")[1].strip("'")
+
+ # If we find a new local dependency, add it.
+ local_dep_match = _LOCAL_DEPENDENCY_PATTERN.search(line)
+ if local_dep_match:
+ local_dependency_name = local_dep_match.group("local_dependency_name")
+ if (
+ local_dependency_name != current_project_name
+ # These lint rules are not part of android-components
+ and local_dependency_name != "mozilla-lint-rules"
+ ):
+ project_dependencies[current_project_name].add(local_dependency_name)
+
+ return {
+ project_name: sorted(project_dependencies[project_name])
+ for project_name in gradle_projects
+ }
+
+
+def _get_gradle_projects(gradle_root, existing_build_config):
+ if gradle_root.endswith("android-components"):
+ return list(existing_build_config["projects"].keys())
+ elif gradle_root.endswith("focus-android"):
+ return ["app"]
+ elif gradle_root.endswith("fenix"):
+ return ["app"]
+
+ raise NotImplementedError(f"Cannot find gradle projects for {gradle_root}")
+
+
+def is_dir(string):
+ if os.path.isdir(string):
+ return string
+ else:
+ raise argparse.ArgumentTypeError(f'"{string}" is not a directory')
+
+
+def _parse_args(cmdln_args):
+ parser = argparse.ArgumentParser(
+ description="Calls gradle and generate json file with dependencies"
+ )
+ parser.add_argument(
+ "gradle_root",
+ metavar="GRADLE_ROOT",
+ type=is_dir,
+ help="The directory where to call gradle from",
+ )
+ return parser.parse_args(args=cmdln_args)
+
+
+def _set_logging_config():
+ logging.basicConfig(
+ level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s"
+ )
+
+
+def _merge_build_config(
+ existing_build_config, upstream_deps_per_project, variants_config
+):
+ updated_build_config = {
+ "projects": {
+ project: {"upstream_dependencies": deps}
+ for project, deps in upstream_deps_per_project.items()
+ }
+ }
+ updated_variant_config = {"variants": variants_config} if variants_config else {}
+ return merge(existing_build_config, updated_build_config, updated_variant_config)
+
+
+def _get_variants(gradle_root):
+ cmd = list(_DEFAULT_GRADLE_COMMAND) + ["printVariants"]
+ output_lines = subprocess.check_output(
+ cmd, universal_newlines=True, cwd=gradle_root
+ ).splitlines()
+ variants_line = [line for line in output_lines if line.startswith("variants: ")][0]
+ variants_json = variants_line.split(" ", 1)[1]
+ return json.loads(variants_json)
+
+
+def _should_print_variants(gradle_root):
+ return gradle_root.endswith("fenix") or gradle_root.endswith("focus-android")
+
+
+def main():
+ args = _parse_args(sys.argv[1:])
+ gradle_root = args.gradle_root
+ build_config_file = os.path.join(gradle_root, ".buildconfig.yml")
+ _set_logging_config()
+
+ with open(build_config_file) as f:
+ existing_build_config = yaml.safe_load(f)
+
+ upstream_deps_per_project = _get_upstream_deps_per_gradle_project(
+ gradle_root, existing_build_config
+ )
+
+ variants_config = (
+ _get_variants(gradle_root) if _should_print_variants(gradle_root) else {}
+ )
+ merged_build_config = _merge_build_config(
+ existing_build_config, upstream_deps_per_project, variants_config
+ )
+
+ with open(build_config_file, "w") as f:
+ yaml.safe_dump(merged_build_config, f)
+ logger.info(f"Updated {build_config_file} with latest gradle config!")
+
+
+__name__ == "__main__" and main()