Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 555 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py
new file mode 100644
index 0000000000..2546e1ae88
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/util/partners.py
@@ -0,0 +1,555 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+import os
+import xml.etree.ElementTree as ET
+from urllib.parse import urlencode
+
+import requests
+import yaml
+from redo import retry
+from taskgraph.util.schema import resolve_keyed_by
+
+from gecko_taskgraph.util.attributes import release_level
+from gecko_taskgraph.util.copy_task import copy_task
+
+# Suppress chatty requests logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+log = logging.getLogger(__name__)
+
+GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+
+"""
+LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
+which users GraphQL. See https://developer.github.com/v4/
+"""
+
+LOGIN_QUERY = """query {
+  viewer {
+    login
+    name
+  }
+}
+"""
+
+# Returns the contents of default.xml from a manifest repository
+MANIFEST_QUERY = """query {
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "master:%(file)s") {
+      ... on Blob {
+        text
+      }
+    }
+  }
+}
+"""
+# Example response:
+# {
+#   "data": {
+#     "repository": {
+#       "object": {
+#         "text": "<?xml version=\"1.0\" ?>\n<manifest>\n  " +
+#           "<remote fetch=\"git@github.com:mozilla-partners/\" name=\"mozilla-partners\"/>\n  " +
+#           "<remote fetch=\"git@github.com:mozilla/\" name=\"mozilla\"/>\n\n  " +
+#           "<project name=\"repack-scripts\" path=\"scripts\" remote=\"mozilla-partners\" " +
+#           "revision=\"master\"/>\n  <project name=\"build-tools\" path=\"scripts/tools\" " +
+#           "remote=\"mozilla\" revision=\"master\"/>\n  <project name=\"mozilla-EME-free\" " +
+#           "path=\"partners/mozilla-EME-free\" remote=\"mozilla-partners\" " +
+#           "revision=\"master\"/>\n</manifest>\n"
+#       }
+#     }
+#   }
+# }
+
+# Returns the contents of desktop/*/repack.cfg for a partner repository
+REPACK_CFG_QUERY = """query{
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "%(revision)s:desktop/"){
+      ... on Tree {
+        entries {
+          name
+          object {
+            ... on Tree {
+              entries {
+                name
+                object {
+                  ... on Blob {
+                    text
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+# Example response:
+# {
+#   "data": {
+#     "repository": {
+#       "object": {
+#         "entries": [
+#           {
+#             "name": "mozilla-EME-free",
+#             "object": {
+#               "entries": [
+#                 {
+#                   "name": "distribution",
+#                   "object": {}
+#                 },
+#                 {
+#                   "name": "repack.cfg",
+#                   "object": {
+#                     "text": "aus=\"mozilla-EMEfree\"\ndist_id=\"mozilla-EMEfree\"\n" +
+#                             "dist_version=\"1.0\"\nlinux-i686=true\nlinux-x86_64=true\n" +
+#                             " locales=\"ach af de en-US\"\nmac=true\nwin32=true\nwin64=true\n" +
+#                             "output_dir=\"%(platform)s-EME-free/%(locale)s\"\n\n" +
+#                             "# Upload params\nbucket=\"net-mozaws-prod-delivery-firefox\"\n" +
+#                             "upload_to_candidates=true\n"
+#                   }
+#                 }
+#               ]
+#             }
+#           }
+#         ]
+#       }
+#     }
+#   }
+# }
+
+# Map platforms in repack.cfg into their equivalents in taskcluster
+TC_PLATFORM_PER_FTP = {
+    "linux-i686": "linux-shippable",
+    "linux-x86_64": "linux64-shippable",
+    "mac": "macosx64-shippable",
+    "win32": "win32-shippable",
+    "win64": "win64-shippable",
+    "win64-aarch64": "win64-aarch64-shippable",
+}
+
+TASKCLUSTER_PROXY_SECRET_ROOT = "http://taskcluster/secrets/v1/secret"
+
+LOCALES_FILE = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
+    "browser",
+    "locales",
+    "l10n-changesets.json",
+)
+
+# cache data at the module level
+partner_configs = {}
+
+
+def get_token(params):
+    """We use a Personal Access Token from Github to lookup partner config. No extra scopes are
+    needed on the token to read public repositories, but need the 'repo' scope to see private
+    repositories. This is not fine grained and also grants r/w access, but is revoked at the repo
+    level.
+    """
+
+    # Allow for local taskgraph debugging
+    if os.environ.get("GITHUB_API_TOKEN"):
+        return os.environ["GITHUB_API_TOKEN"]
+
+    # The 'usual' method - via taskClusterProxy for decision tasks
+    url = "{secret_root}/project/releng/gecko/build/level-{level}/partner-github-api".format(
+        secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params
+    )
+    try:
+        resp = retry(
+            requests.get,
+            attempts=2,
+            sleeptime=10,
+            args=(url,),
+            kwargs={"timeout": 60, "headers": ""},
+        )
+        j = resp.json()
+        return j["secret"]["key"]
+    except (requests.ConnectionError, ValueError, KeyError):
+        raise RuntimeError("Could not get Github API token to lookup partner data")
+
+
+def query_api(query, token):
+    """Make a query with a Github auth header, returning the json"""
+    headers = {"Authorization": "bearer %s" % token}
+    r = requests.post(GITHUB_API_ENDPOINT, json={"query": query}, headers=headers)
+    r.raise_for_status()
+
+    j = r.json()
+    if "errors" in j:
+        raise RuntimeError("Github query error - %s", j["errors"])
+    return j
+
+
+def check_login(token):
+    log.debug("Checking we have a valid login")
+    query_api(LOGIN_QUERY, token)
+
+
+def get_repo_params(repo):
+    """Parse the organisation and repo name from an https or git url for a repo"""
+    if repo.startswith("https"):
+        # eg https://github.com/mozilla-partners/mozilla-EME-free
+        return repo.rsplit("/", 2)[-2:]
+    if repo.startswith("git@"):
+        # eg git@github.com:mozilla-partners/mailru.git
+        repo = repo.replace(".git", "")
+        return repo.split(":")[-1].split("/")
+
+
+def get_partners(manifestRepo, token):
+    """Given the url to a manifest repository, retrieve the default.xml and parse it into a
+    list of partner repos.
+    """
+    log.debug("Querying for manifest default.xml in %s", manifestRepo)
+    owner, repo = get_repo_params(manifestRepo)
+    query = MANIFEST_QUERY % {"owner": owner, "repo": repo, "file": "default.xml"}
+    raw_manifest = query_api(query, token)
+    log.debug("Raw manifest: %s", raw_manifest)
+    if not raw_manifest["data"]["repository"]:
+        raise RuntimeError(
+            "Couldn't load partner manifest at %s, insufficient permissions ?"
+            % manifestRepo
+        )
+    e = ET.fromstring(raw_manifest["data"]["repository"]["object"]["text"])
+
+    remotes = {}
+    partners = {}
+    for child in e:
+        if child.tag == "remote":
+            name = child.attrib["name"]
+            url = child.attrib["fetch"]
+            remotes[name] = url
+            log.debug("Added remote %s at %s", name, url)
+        elif child.tag == "project":
+            # we don't need to check any code repos
+            if "scripts" in child.attrib["path"]:
+                continue
+            owner, _ = get_repo_params(remotes[child.attrib["remote"]] + "_")
+            partner_url = {
+                "owner": owner,
+                "repo": child.attrib["name"],
+                "revision": child.attrib["revision"],
+            }
+            partners[child.attrib["name"]] = partner_url
+            log.debug(
+                "Added partner %s at revision %s"
+                % (partner_url["repo"], partner_url["revision"])
+            )
+    return partners
+
+
+def parse_config(data):
+    """Parse a single repack.cfg file into a python dictionary.
+    data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
+    locales and platforms data, otherwise passthrough
+    """
+    ALLOWED_KEYS = (
+        "locales",
+        "platforms",
+        "upload_to_candidates",
+        "repack_stub_installer",
+        "publish_to_releases",
+    )
+    config = {"platforms": []}
+    for l in data.splitlines():
+        if "=" in l:
+            l = str(l)
+            key, value = l.split("=", 1)
+            value = value.strip("'\"").rstrip("'\"")
+            if key in TC_PLATFORM_PER_FTP.keys():
+                if value.lower() == "true":
+                    config["platforms"].append(TC_PLATFORM_PER_FTP[key])
+                continue
+            if key not in ALLOWED_KEYS:
+                continue
+            if key == "locales":
+                # a list please
+                value = value.split(" ")
+            config[key] = value
+    return config
+
+
+def get_repack_configs(repackRepo, token):
+    """For a partner repository, retrieve all the repack.cfg files and parse them into a dict"""
+    log.debug("Querying for configs in %s", repackRepo)
+    query = REPACK_CFG_QUERY % repackRepo
+    raw_configs = query_api(query, token)
+    raw_configs = raw_configs["data"]["repository"]["object"]["entries"]
+
+    configs = {}
+    for sub_config in raw_configs:
+        name = sub_config["name"]
+        for file in sub_config["object"].get("entries", []):
+            if file["name"] != "repack.cfg":
+                continue
+            configs[name] = parse_config(file["object"]["text"])
+    return configs
+
+
+def get_attribution_config(manifestRepo, token):
+    log.debug("Querying for manifest attribution_config.yml in %s", manifestRepo)
+    owner, repo = get_repo_params(manifestRepo)
+    query = MANIFEST_QUERY % {
+        "owner": owner,
+        "repo": repo,
+        "file": "attribution_config.yml",
+    }
+    raw_manifest = query_api(query, token)
+    if not raw_manifest["data"]["repository"]:
+        raise RuntimeError(
+            "Couldn't load partner manifest at %s, insufficient permissions ?"
+            % manifestRepo
+        )
+    # no file has been set up, gracefully continue
+    if raw_manifest["data"]["repository"]["object"] is None:
+        log.debug("No attribution_config.yml file found")
+        return {}
+
+    return yaml.safe_load(raw_manifest["data"]["repository"]["object"]["text"])
+
+
+def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
+    """Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    If partner_subset is a list of sub_config names only return data for those.
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    if not manifest_url:
+        raise RuntimeError(f"Manifest url for {kind} not defined")
+    if kind not in partner_configs:
+        log.info("Looking up data for %s from %s", kind, manifest_url)
+        check_login(token)
+        if kind == "release-partner-attribution":
+            partner_configs[kind] = get_attribution_config(manifest_url, token)
+        else:
+            partners = get_partners(manifest_url, token)
+
+            partner_configs[kind] = {}
+            for partner, partner_url in partners.items():
+                if partner_subset and partner not in partner_subset:
+                    continue
+                partner_configs[kind][partner] = get_repack_configs(partner_url, token)
+
+    return partner_configs[kind]
+
+
+def check_if_partners_enabled(config, tasks):
+    if (
+        (
+            config.params["release_enable_partner_repack"]
+            and config.kind.startswith("release-partner-repack")
+        )
+        or (
+            config.params["release_enable_partner_attribution"]
+            and config.kind.startswith("release-partner-attribution")
+        )
+        or (
+            config.params["release_enable_emefree"]
+            and config.kind.startswith("release-eme-free-")
+        )
+    ):
+        yield from tasks
+
+
+def get_partner_config_by_kind(config, kind):
+    """Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    partner_subset = config.params["release_partners"]
+    partner_configs = config.params["release_partner_config"] or {}
+
+    # TODO eme-free should be a partner; we shouldn't care about per-kind
+    for k in partner_configs:
+        if kind.startswith(k):
+            kind_config = partner_configs[k]
+            break
+    else:
+        return {}
+    # if we're only interested in a subset of partners we remove the rest
+    if partner_subset:
+        if kind.startswith("release-partner-repack"):
+            # TODO - should be fatal to have an unknown partner in partner_subset
+            for partner in [p for p in kind_config.keys() if p not in partner_subset]:
+                del kind_config[partner]
+        elif kind.startswith("release-partner-attribution") and isinstance(
+            kind_config, dict
+        ):
+            all_configs = copy_task(kind_config.get("configs", []))
+            kind_config["configs"] = []
+            for this_config in all_configs:
+                if this_config["campaign"] in partner_subset:
+                    kind_config["configs"].append(this_config)
+    return kind_config
+
+
+def _fix_subpartner_locales(orig_config, all_locales):
+    subpartner_config = copy_task(orig_config)
+    # Get an ordered list of subpartner locales that is a subset of all_locales
+    subpartner_config["locales"] = sorted(
+        list(set(orig_config["locales"]) & set(all_locales))
+    )
+    return subpartner_config
+
+
+def fix_partner_config(orig_config):
+    pc = {}
+    with open(LOCALES_FILE) as fh:
+        all_locales = list(json.load(fh).keys())
+    # l10n-changesets.json doesn't include en-US, but the repack list does
+    if "en-US" not in all_locales:
+        all_locales.append("en-US")
+    for kind, kind_config in orig_config.items():
+        if kind == "release-partner-attribution":
+            pc[kind] = {}
+            if kind_config:
+                pc[kind] = {"defaults": kind_config["defaults"]}
+                for config in kind_config["configs"]:
+                    # Make sure our locale list is a subset of all_locales
+                    pc[kind].setdefault("configs", []).append(
+                        _fix_subpartner_locales(config, all_locales)
+                    )
+        else:
+            for partner, partner_config in kind_config.items():
+                for subpartner, subpartner_config in partner_config.items():
+                    # get rid of empty subpartner configs
+                    if not subpartner_config:
+                        continue
+                    # Make sure our locale list is a subset of all_locales
+                    pc.setdefault(kind, {}).setdefault(partner, {})[
+                        subpartner
+                    ] = _fix_subpartner_locales(subpartner_config, all_locales)
+    return pc
+
+
+# seems likely this exists elsewhere already
+def get_ftp_platform(platform):
+    if platform.startswith("win32"):
+        return "win32"
+    if platform.startswith("win64-aarch64"):
+        return "win64-aarch64"
+    if platform.startswith("win64"):
+        return "win64"
+    if platform.startswith("macosx"):
+        return "mac"
+    if platform.startswith("linux-"):
+        return "linux-i686"
+    if platform.startswith("linux64"):
+        return "linux-x86_64"
+    raise ValueError(f"Unimplemented platform {platform}")
+
+
+# Ugh
+def locales_per_build_platform(build_platform, locales):
+    if build_platform.startswith("mac"):
+        exclude = ["ja"]
+    else:
+        exclude = ["ja-JP-mac"]
+    return [locale for locale in locales if locale not in exclude]
+
+
+def get_partner_url_config(parameters, graph_config):
+    partner_url_config = copy_task(graph_config["partner-urls"])
+    substitutions = {
+        "release-product": parameters["release_product"],
+        "release-level": release_level(parameters["project"]),
+        "release-type": parameters["release_type"],
+    }
+    resolve_keyed_by(
+        partner_url_config,
+        "release-eme-free-repack",
+        "eme-free manifest_url",
+        **substitutions,
+    )
+    resolve_keyed_by(
+        partner_url_config,
+        "release-partner-repack",
+        "partner manifest url",
+        **substitutions,
+    )
+    resolve_keyed_by(
+        partner_url_config,
+        "release-partner-attribution",
+        "partner attribution url",
+        **substitutions,
+    )
+    return partner_url_config
+
+
+def get_repack_ids_by_platform(config, build_platform):
+    partner_config = get_partner_config_by_kind(config, config.kind)
+    combinations = []
+    for partner, subconfigs in partner_config.items():
+        for sub_config_name, sub_config in subconfigs.items():
+            if build_platform not in sub_config.get("platforms", []):
+                continue
+            locales = locales_per_build_platform(
+                build_platform, sub_config.get("locales", [])
+            )
+            for locale in locales:
+                combinations.append(f"{partner}/{sub_config_name}/{locale}")
+    return sorted(combinations)
+
+
+def get_partners_to_be_published(config):
+    # hardcoded kind because release-bouncer-aliases doesn't match otherwise
+    partner_config = get_partner_config_by_kind(config, "release-partner-repack")
+    partners = []
+    for partner, subconfigs in partner_config.items():
+        for sub_config_name, sub_config in subconfigs.items():
+            if sub_config.get("publish_to_releases"):
+                partners.append((partner, sub_config_name, sub_config["platforms"]))
+    return partners
+
+
+def apply_partner_priority(config, jobs):
+    priority = None
+    # Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile
+    # leave EME-free jobs alone because they do, and they'll get the branch priority like the rest
+    # of the release. Only bother with this in production, not on staging releases on try.
+    # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than
+    # integration branches because we don't want to wait a lot for the graph to be done, but
+    # for multiple releases the partner tasks always wait for non-partner.
+    if (
+        config.kind.startswith(
+            ("release-partner-repack", "release-partner-attribution")
+        )
+        and release_level(config.params["project"]) == "production"
+    ):
+        priority = "medium"
+    for job in jobs:
+        if priority:
+            job["priority"] = priority
+        yield job
+
+
+def generate_attribution_code(defaults, partner):
+    params = {
+        "medium": defaults["medium"],
+        "source": defaults["source"],
+        "campaign": partner["campaign"],
+        "content": partner["content"],
+    }
+    if partner.get("variation"):
+        params["variation"] = partner["variation"]
+    if partner.get("experiment"):
+        params["experiment"] = partner["experiment"]
+
+    code = urlencode(params)
+    return code