summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/util/partners.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /taskcluster/gecko_taskgraph/util/partners.py
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/gecko_taskgraph/util/partners.py')
-rw-r--r--taskcluster/gecko_taskgraph/util/partners.py555
1 files changed, 555 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py
new file mode 100644
index 0000000000..2546e1ae88
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/util/partners.py
@@ -0,0 +1,555 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+import os
+import xml.etree.ElementTree as ET
+from urllib.parse import urlencode
+
+import requests
+import yaml
+from redo import retry
+from taskgraph.util.schema import resolve_keyed_by
+
+from gecko_taskgraph.util.attributes import release_level
+from gecko_taskgraph.util.copy_task import copy_task
+
+# Suppress chatty requests logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+log = logging.getLogger(__name__)
+
+GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+
+"""
+LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
+which users GraphQL. See https://developer.github.com/v4/
+"""
+
+LOGIN_QUERY = """query {
+ viewer {
+ login
+ name
+ }
+}
+"""
+
+# Returns the contents of default.xml from a manifest repository
+MANIFEST_QUERY = """query {
+ repository(owner:"%(owner)s", name:"%(repo)s") {
+ object(expression: "master:%(file)s") {
+ ... on Blob {
+ text
+ }
+ }
+ }
+}
+"""
+# Example response:
+# {
+# "data": {
+# "repository": {
+# "object": {
+# "text": "<?xml version=\"1.0\" ?>\n<manifest>\n " +
+# "<remote fetch=\"git@github.com:mozilla-partners/\" name=\"mozilla-partners\"/>\n " +
+# "<remote fetch=\"git@github.com:mozilla/\" name=\"mozilla\"/>\n\n " +
+# "<project name=\"repack-scripts\" path=\"scripts\" remote=\"mozilla-partners\" " +
+# "revision=\"master\"/>\n <project name=\"build-tools\" path=\"scripts/tools\" " +
+# "remote=\"mozilla\" revision=\"master\"/>\n <project name=\"mozilla-EME-free\" " +
+# "path=\"partners/mozilla-EME-free\" remote=\"mozilla-partners\" " +
+# "revision=\"master\"/>\n</manifest>\n"
+# }
+# }
+# }
+# }
+
+# Returns the contents of desktop/*/repack.cfg for a partner repository
+REPACK_CFG_QUERY = """query{
+ repository(owner:"%(owner)s", name:"%(repo)s") {
+ object(expression: "%(revision)s:desktop/"){
+ ... on Tree {
+ entries {
+ name
+ object {
+ ... on Tree {
+ entries {
+ name
+ object {
+ ... on Blob {
+ text
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+"""
+# Example response:
+# {
+# "data": {
+# "repository": {
+# "object": {
+# "entries": [
+# {
+# "name": "mozilla-EME-free",
+# "object": {
+# "entries": [
+# {
+# "name": "distribution",
+# "object": {}
+# },
+# {
+# "name": "repack.cfg",
+# "object": {
+# "text": "aus=\"mozilla-EMEfree\"\ndist_id=\"mozilla-EMEfree\"\n" +
+# "dist_version=\"1.0\"\nlinux-i686=true\nlinux-x86_64=true\n" +
+# " locales=\"ach af de en-US\"\nmac=true\nwin32=true\nwin64=true\n" +
+# "output_dir=\"%(platform)s-EME-free/%(locale)s\"\n\n" +
+# "# Upload params\nbucket=\"net-mozaws-prod-delivery-firefox\"\n" +
+# "upload_to_candidates=true\n"
+# }
+# }
+# ]
+# }
+# }
+# ]
+# }
+# }
+# }
+# }
+
+# Map platforms in repack.cfg into their equivalents in taskcluster
+TC_PLATFORM_PER_FTP = {
+ "linux-i686": "linux-shippable",
+ "linux-x86_64": "linux64-shippable",
+ "mac": "macosx64-shippable",
+ "win32": "win32-shippable",
+ "win64": "win64-shippable",
+ "win64-aarch64": "win64-aarch64-shippable",
+}
+
+TASKCLUSTER_PROXY_SECRET_ROOT = "http://taskcluster/secrets/v1/secret"
+
+LOCALES_FILE = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
+ "browser",
+ "locales",
+ "l10n-changesets.json",
+)
+
+# cache data at the module level
+partner_configs = {}
+
+
+def get_token(params):
+ """We use a Personal Access Token from Github to lookup partner config. No extra scopes are
+ needed on the token to read public repositories, but need the 'repo' scope to see private
+ repositories. This is not fine grained and also grants r/w access, but is revoked at the repo
+ level.
+ """
+
+ # Allow for local taskgraph debugging
+ if os.environ.get("GITHUB_API_TOKEN"):
+ return os.environ["GITHUB_API_TOKEN"]
+
+ # The 'usual' method - via taskClusterProxy for decision tasks
+ url = "{secret_root}/project/releng/gecko/build/level-{level}/partner-github-api".format(
+ secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params
+ )
+ try:
+ resp = retry(
+ requests.get,
+ attempts=2,
+ sleeptime=10,
+ args=(url,),
+ kwargs={"timeout": 60, "headers": ""},
+ )
+ j = resp.json()
+ return j["secret"]["key"]
+ except (requests.ConnectionError, ValueError, KeyError):
+ raise RuntimeError("Could not get Github API token to lookup partner data")
+
+
+def query_api(query, token):
+ """Make a query with a Github auth header, returning the json"""
+ headers = {"Authorization": "bearer %s" % token}
+ r = requests.post(GITHUB_API_ENDPOINT, json={"query": query}, headers=headers)
+ r.raise_for_status()
+
+ j = r.json()
+ if "errors" in j:
+ raise RuntimeError("Github query error - %s", j["errors"])
+ return j
+
+
+def check_login(token):
+ log.debug("Checking we have a valid login")
+ query_api(LOGIN_QUERY, token)
+
+
+def get_repo_params(repo):
+ """Parse the organisation and repo name from an https or git url for a repo"""
+ if repo.startswith("https"):
+ # eg https://github.com/mozilla-partners/mozilla-EME-free
+ return repo.rsplit("/", 2)[-2:]
+ if repo.startswith("git@"):
+ # eg git@github.com:mozilla-partners/mailru.git
+ repo = repo.replace(".git", "")
+ return repo.split(":")[-1].split("/")
+
+
+def get_partners(manifestRepo, token):
+ """Given the url to a manifest repository, retrieve the default.xml and parse it into a
+ list of partner repos.
+ """
+ log.debug("Querying for manifest default.xml in %s", manifestRepo)
+ owner, repo = get_repo_params(manifestRepo)
+ query = MANIFEST_QUERY % {"owner": owner, "repo": repo, "file": "default.xml"}
+ raw_manifest = query_api(query, token)
+ log.debug("Raw manifest: %s", raw_manifest)
+ if not raw_manifest["data"]["repository"]:
+ raise RuntimeError(
+ "Couldn't load partner manifest at %s, insufficient permissions ?"
+ % manifestRepo
+ )
+ e = ET.fromstring(raw_manifest["data"]["repository"]["object"]["text"])
+
+ remotes = {}
+ partners = {}
+ for child in e:
+ if child.tag == "remote":
+ name = child.attrib["name"]
+ url = child.attrib["fetch"]
+ remotes[name] = url
+ log.debug("Added remote %s at %s", name, url)
+ elif child.tag == "project":
+ # we don't need to check any code repos
+ if "scripts" in child.attrib["path"]:
+ continue
+ owner, _ = get_repo_params(remotes[child.attrib["remote"]] + "_")
+ partner_url = {
+ "owner": owner,
+ "repo": child.attrib["name"],
+ "revision": child.attrib["revision"],
+ }
+ partners[child.attrib["name"]] = partner_url
+ log.debug(
+ "Added partner %s at revision %s"
+ % (partner_url["repo"], partner_url["revision"])
+ )
+ return partners
+
+
+def parse_config(data):
+ """Parse a single repack.cfg file into a python dictionary.
+ data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
+ locales and platforms data, otherwise passthrough
+ """
+ ALLOWED_KEYS = (
+ "locales",
+ "platforms",
+ "upload_to_candidates",
+ "repack_stub_installer",
+ "publish_to_releases",
+ )
+ config = {"platforms": []}
+ for l in data.splitlines():
+ if "=" in l:
+ l = str(l)
+ key, value = l.split("=", 1)
+ value = value.strip("'\"").rstrip("'\"")
+ if key in TC_PLATFORM_PER_FTP.keys():
+ if value.lower() == "true":
+ config["platforms"].append(TC_PLATFORM_PER_FTP[key])
+ continue
+ if key not in ALLOWED_KEYS:
+ continue
+ if key == "locales":
+ # a list please
+ value = value.split(" ")
+ config[key] = value
+ return config
+
+
+def get_repack_configs(repackRepo, token):
+ """For a partner repository, retrieve all the repack.cfg files and parse them into a dict"""
+ log.debug("Querying for configs in %s", repackRepo)
+ query = REPACK_CFG_QUERY % repackRepo
+ raw_configs = query_api(query, token)
+ raw_configs = raw_configs["data"]["repository"]["object"]["entries"]
+
+ configs = {}
+ for sub_config in raw_configs:
+ name = sub_config["name"]
+ for file in sub_config["object"].get("entries", []):
+ if file["name"] != "repack.cfg":
+ continue
+ configs[name] = parse_config(file["object"]["text"])
+ return configs
+
+
+def get_attribution_config(manifestRepo, token):
+ log.debug("Querying for manifest attribution_config.yml in %s", manifestRepo)
+ owner, repo = get_repo_params(manifestRepo)
+ query = MANIFEST_QUERY % {
+ "owner": owner,
+ "repo": repo,
+ "file": "attribution_config.yml",
+ }
+ raw_manifest = query_api(query, token)
+ if not raw_manifest["data"]["repository"]:
+ raise RuntimeError(
+ "Couldn't load partner manifest at %s, insufficient permissions ?"
+ % manifestRepo
+ )
+ # no file has been set up, gracefully continue
+ if raw_manifest["data"]["repository"]["object"] is None:
+ log.debug("No attribution_config.yml file found")
+ return {}
+
+ return yaml.safe_load(raw_manifest["data"]["repository"]["object"]["text"])
+
+
+def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
+ """Retrieve partner data starting from the manifest url, which points to a repository
+ containing a default.xml that is intended to be drive the Google tool 'repo'. It
+ descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+ If partner_subset is a list of sub_config names only return data for those.
+
+ Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+ partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+ """
+ if not manifest_url:
+ raise RuntimeError(f"Manifest url for {kind} not defined")
+ if kind not in partner_configs:
+ log.info("Looking up data for %s from %s", kind, manifest_url)
+ check_login(token)
+ if kind == "release-partner-attribution":
+ partner_configs[kind] = get_attribution_config(manifest_url, token)
+ else:
+ partners = get_partners(manifest_url, token)
+
+ partner_configs[kind] = {}
+ for partner, partner_url in partners.items():
+ if partner_subset and partner not in partner_subset:
+ continue
+ partner_configs[kind][partner] = get_repack_configs(partner_url, token)
+
+ return partner_configs[kind]
+
+
+def check_if_partners_enabled(config, tasks):
+ if (
+ (
+ config.params["release_enable_partner_repack"]
+ and config.kind.startswith("release-partner-repack")
+ )
+ or (
+ config.params["release_enable_partner_attribution"]
+ and config.kind.startswith("release-partner-attribution")
+ )
+ or (
+ config.params["release_enable_emefree"]
+ and config.kind.startswith("release-eme-free-")
+ )
+ ):
+ yield from tasks
+
+
+def get_partner_config_by_kind(config, kind):
+ """Retrieve partner data starting from the manifest url, which points to a repository
+ containing a default.xml that is intended to be drive the Google tool 'repo'. It
+ descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+ Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+ partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+ """
+ partner_subset = config.params["release_partners"]
+ partner_configs = config.params["release_partner_config"] or {}
+
+ # TODO eme-free should be a partner; we shouldn't care about per-kind
+ for k in partner_configs:
+ if kind.startswith(k):
+ kind_config = partner_configs[k]
+ break
+ else:
+ return {}
+ # if we're only interested in a subset of partners we remove the rest
+ if partner_subset:
+ if kind.startswith("release-partner-repack"):
+ # TODO - should be fatal to have an unknown partner in partner_subset
+ for partner in [p for p in kind_config.keys() if p not in partner_subset]:
+ del kind_config[partner]
+ elif kind.startswith("release-partner-attribution") and isinstance(
+ kind_config, dict
+ ):
+ all_configs = copy_task(kind_config.get("configs", []))
+ kind_config["configs"] = []
+ for this_config in all_configs:
+ if this_config["campaign"] in partner_subset:
+ kind_config["configs"].append(this_config)
+ return kind_config
+
+
+def _fix_subpartner_locales(orig_config, all_locales):
+ subpartner_config = copy_task(orig_config)
+ # Get an ordered list of subpartner locales that is a subset of all_locales
+ subpartner_config["locales"] = sorted(
+ list(set(orig_config["locales"]) & set(all_locales))
+ )
+ return subpartner_config
+
+
+def fix_partner_config(orig_config):
+ pc = {}
+ with open(LOCALES_FILE) as fh:
+ all_locales = list(json.load(fh).keys())
+ # l10n-changesets.json doesn't include en-US, but the repack list does
+ if "en-US" not in all_locales:
+ all_locales.append("en-US")
+ for kind, kind_config in orig_config.items():
+ if kind == "release-partner-attribution":
+ pc[kind] = {}
+ if kind_config:
+ pc[kind] = {"defaults": kind_config["defaults"]}
+ for config in kind_config["configs"]:
+ # Make sure our locale list is a subset of all_locales
+ pc[kind].setdefault("configs", []).append(
+ _fix_subpartner_locales(config, all_locales)
+ )
+ else:
+ for partner, partner_config in kind_config.items():
+ for subpartner, subpartner_config in partner_config.items():
+ # get rid of empty subpartner configs
+ if not subpartner_config:
+ continue
+ # Make sure our locale list is a subset of all_locales
+ pc.setdefault(kind, {}).setdefault(partner, {})[
+ subpartner
+ ] = _fix_subpartner_locales(subpartner_config, all_locales)
+ return pc
+
+
+# seems likely this exists elsewhere already
+def get_ftp_platform(platform):
+ if platform.startswith("win32"):
+ return "win32"
+ if platform.startswith("win64-aarch64"):
+ return "win64-aarch64"
+ if platform.startswith("win64"):
+ return "win64"
+ if platform.startswith("macosx"):
+ return "mac"
+ if platform.startswith("linux-"):
+ return "linux-i686"
+ if platform.startswith("linux64"):
+ return "linux-x86_64"
+ raise ValueError(f"Unimplemented platform {platform}")
+
+
+# Ugh
+def locales_per_build_platform(build_platform, locales):
+ if build_platform.startswith("mac"):
+ exclude = ["ja"]
+ else:
+ exclude = ["ja-JP-mac"]
+ return [locale for locale in locales if locale not in exclude]
+
+
+def get_partner_url_config(parameters, graph_config):
+ partner_url_config = copy_task(graph_config["partner-urls"])
+ substitutions = {
+ "release-product": parameters["release_product"],
+ "release-level": release_level(parameters["project"]),
+ "release-type": parameters["release_type"],
+ }
+ resolve_keyed_by(
+ partner_url_config,
+ "release-eme-free-repack",
+ "eme-free manifest_url",
+ **substitutions,
+ )
+ resolve_keyed_by(
+ partner_url_config,
+ "release-partner-repack",
+ "partner manifest url",
+ **substitutions,
+ )
+ resolve_keyed_by(
+ partner_url_config,
+ "release-partner-attribution",
+ "partner attribution url",
+ **substitutions,
+ )
+ return partner_url_config
+
+
+def get_repack_ids_by_platform(config, build_platform):
+ partner_config = get_partner_config_by_kind(config, config.kind)
+ combinations = []
+ for partner, subconfigs in partner_config.items():
+ for sub_config_name, sub_config in subconfigs.items():
+ if build_platform not in sub_config.get("platforms", []):
+ continue
+ locales = locales_per_build_platform(
+ build_platform, sub_config.get("locales", [])
+ )
+ for locale in locales:
+ combinations.append(f"{partner}/{sub_config_name}/{locale}")
+ return sorted(combinations)
+
+
+def get_partners_to_be_published(config):
+ # hardcoded kind because release-bouncer-aliases doesn't match otherwise
+ partner_config = get_partner_config_by_kind(config, "release-partner-repack")
+ partners = []
+ for partner, subconfigs in partner_config.items():
+ for sub_config_name, sub_config in subconfigs.items():
+ if sub_config.get("publish_to_releases"):
+ partners.append((partner, sub_config_name, sub_config["platforms"]))
+ return partners
+
+
+def apply_partner_priority(config, jobs):
+ priority = None
+ # Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile
+ # leave EME-free jobs alone because they do, and they'll get the branch priority like the rest
+ # of the release. Only bother with this in production, not on staging releases on try.
+ # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than
+ # integration branches because we don't want to wait a lot for the graph to be done, but
+ # for multiple releases the partner tasks always wait for non-partner.
+ if (
+ config.kind.startswith(
+ ("release-partner-repack", "release-partner-attribution")
+ )
+ and release_level(config.params["project"]) == "production"
+ ):
+ priority = "medium"
+ for job in jobs:
+ if priority:
+ job["priority"] = priority
+ yield job
+
+
+def generate_attribution_code(defaults, partner):
+ params = {
+ "medium": defaults["medium"],
+ "source": defaults["source"],
+ "campaign": partner["campaign"],
+ "content": partner["content"],
+ }
+ if partner.get("variation"):
+ params["variation"] = partner["variation"]
+ if partner.get("experiment"):
+ params["experiment"] = partner["experiment"]
+
+ code = urlencode(params)
+ return code