Merging upstream version 127.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

17 files changed, 227 insertions, 259 deletions

diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/archive.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/archive.py
index ee59ba4548..261a031038 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/archive.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/archive.py
@@ -12,6 +12,40 @@ import tarfile
 DEFAULT_MTIME = 1451606400
 
 
+# Python 3.9 contains this change:
+#  https://github.com/python/cpython/commit/674935b8caf33e47c78f1b8e197b1b77a04992d2
+# which changes the output of tar creation compared to earlier versions.
+# As this code is used to generate tar files that are meant to be deterministic
+# across versions of python (specifically, it's used as part of computing the hash
+# of docker images, which needs to be identical between CI (which uses python 3.8),
+# and developer environments (using arbitrary versions of python, at this point,
+# most probably more recent than 3.9)).
+# What we do is subblass TarInfo so that if used on python >= 3.9, it reproduces the
+# behavior from python < 3.9.
+# Here's how it goes:
+# - the behavior in python >= 3.9 is the same as python < 3.9 when the type encoded
+# in the tarinfo is CHRTYPE or BLKTYPE.
+# - the value of the type is only compared in the context of choosing which behavior
+# to take
+# - we replace the type with the same value (so that using the value has no changes)
+# but that pretends to be the same as CHRTYPE so that the condition that enables the
+# old behavior is taken.
+class HackedType(bytes):
+    def __eq__(self, other):
+        if other == tarfile.CHRTYPE:
+            return True
+        return self == other
+
+
+class TarInfo(tarfile.TarInfo):
+    @staticmethod
+    def _create_header(info, format, encoding, errors):
+        info["type"] = HackedType(info["type"])
+        # ignore type checking because it looks like pyright complains because we're calling a
+        # non-public method
+        return tarfile.TarInfo._create_header(info, format, encoding, errors)  # type: ignore
+
+
 def create_tar_from_files(fp, files):
     """Create a tar file deterministically.
 
@@ -25,15 +59,23 @@ def create_tar_from_files(fp, files):
 
     FUTURE accept a filename argument (or create APIs to write files)
     """
-    with tarfile.open(name="", mode="w", fileobj=fp, dereference=True) as tf:
+    # The format is explicitly set to tarfile.GNU_FORMAT, because this default format
+    # has been changed in Python 3.8.
+    with tarfile.open(
+        name="", mode="w", fileobj=fp, dereference=True, format=tarfile.GNU_FORMAT
+    ) as tf:
         for archive_path, f in sorted(files.items()):
             if isinstance(f, str):
-                mode = os.stat(f).st_mode
+                s = os.stat(f)
+                mode = s.st_mode
+                size = s.st_size
                 f = open(f, "rb")
             else:
                 mode = 0o0644
+                size = len(f.read())
+                f.seek(0)
 
-            ti = tarfile.TarInfo(archive_path)
+            ti = TarInfo(archive_path)
             ti.mode = mode
             ti.type = tarfile.REGTYPE
 
@@ -56,9 +98,7 @@ def create_tar_from_files(fp, files):
             # Set mtime to a constant value.
             ti.mtime = DEFAULT_MTIME
 
-            f.seek(0, 2)
-            ti.size = f.tell()
-            f.seek(0, 0)
+            ti.size = size
             # tarfile wants to pass a size argument to read(). So just
             # wrap/buffer in a proper file object interface.
             tf.addfile(ti, f)
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/cached_tasks.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/cached_tasks.py
index 974b114902..1a3baad5be 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/cached_tasks.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/cached_tasks.py
@@ -7,6 +7,7 @@ import hashlib
 import time
 
 TARGET_CACHE_INDEX = "{cache_prefix}.cache.level-{level}.{type}.{name}.hash.{digest}"
+TARGET_PR_CACHE_INDEX = "{cache_prefix}.cache.pr.{type}.{name}.hash.{digest}"
 EXTRA_CACHE_INDEXES = [
     "{cache_prefix}.cache.level-{level}.{type}.{name}.latest",
     "{cache_prefix}.cache.level-{level}.{type}.{name}.pushdate.{build_date_long}",
@@ -53,31 +54,45 @@ def add_optimization(
 
     # We'll try to find a cached version of the toolchain at levels above and
     # including the current level, starting at the highest level.
-    # Chain-of-trust doesn't handle tasks not built on the tip of a
-    # pull-request, so don't look for level-1 tasks if building a pull-request.
     index_routes = []
     min_level = int(config.params["level"])
-    if config.params["tasks_for"] == "github-pull-request":
-        min_level = max(min_level, 3)
     for level in reversed(range(min_level, 4)):
         subs["level"] = level
         index_routes.append(TARGET_CACHE_INDEX.format(**subs))
 
-        taskdesc["optimization"] = {"index-search": index_routes}
+    # Pull requests use a different target cache index route. This way we can
+    # be confident they won't be used by anything other than the pull request
+    # that created the cache in the first place.
+    if config.params["tasks_for"].startswith(
+        "github-pull-request"
+    ) and config.graph_config["taskgraph"].get("cache-pull-requests", True):
+        subs["head_ref"] = config.params["head_ref"]
+        if subs["head_ref"].startswith("refs/heads/"):
+            subs["head_ref"] = subs["head_ref"][11:]
+        index_routes.append(TARGET_PR_CACHE_INDEX.format(**subs))
+
+    taskdesc["optimization"] = {"index-search": index_routes}
 
     # ... and cache at the lowest level.
     subs["level"] = config.params["level"]
-    taskdesc.setdefault("routes", []).append(
-        f"index.{TARGET_CACHE_INDEX.format(**subs)}"
-    )
 
-    # ... and add some extra routes for humans
-    subs["build_date_long"] = time.strftime(
-        "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"])
-    )
-    taskdesc["routes"].extend(
-        [f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES]
-    )
+    if config.params["tasks_for"].startswith("github-pull-request"):
+        if config.graph_config["taskgraph"].get("cache-pull-requests", True):
+            taskdesc.setdefault("routes", []).append(
+                f"index.{TARGET_PR_CACHE_INDEX.format(**subs)}"
+            )
+    else:
+        taskdesc.setdefault("routes", []).append(
+            f"index.{TARGET_CACHE_INDEX.format(**subs)}"
+        )
+
+        # ... and add some extra routes for humans
+        subs["build_date_long"] = time.strftime(
+            "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"])
+        )
+        taskdesc["routes"].extend(
+            [f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES]
+        )
 
     taskdesc["attributes"]["cached_task"] = {
         "type": cache_type,
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/decision.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/decision.py
deleted file mode 100644
index d0e1e1079f..0000000000
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/decision.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-Utilities for generating a decision task from :file:`.taskcluster.yml`.
-"""
-
-
-import os
-
-import jsone
-import slugid
-import yaml
-
-from .templates import merge
-from .time import current_json_time
-from .vcs import find_hg_revision_push_info
-
-
-def make_decision_task(params, root, context, head_rev=None):
-    """Generate a basic decision task, based on the root .taskcluster.yml"""
-    with open(os.path.join(root, ".taskcluster.yml"), "rb") as f:
-        taskcluster_yml = yaml.safe_load(f)
-
-    if not head_rev:
-        head_rev = params["head_rev"]
-
-    if params["repository_type"] == "hg":
-        pushlog = find_hg_revision_push_info(params["repository_url"], head_rev)
-
-        hg_push_context = {
-            "pushlog_id": pushlog["pushid"],
-            "pushdate": pushlog["pushdate"],
-            "owner": pushlog["user"],
-        }
-    else:
-        hg_push_context = {}
-
-    slugids = {}
-
-    def as_slugid(name):
-        # https://github.com/taskcluster/json-e/issues/164
-        name = name[0]
-        if name not in slugids:
-            slugids[name] = slugid.nice()
-        return slugids[name]
-
-    # provide a similar JSON-e context to what mozilla-taskcluster provides:
-    # https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
-    # but with a different tasks_for and an extra `cron` section
-    context = merge(
-        {
-            "repository": {
-                "url": params["repository_url"],
-                "project": params["project"],
-                "level": params["level"],
-            },
-            "push": merge(
-                {
-                    "revision": params["head_rev"],
-                    # remainder are fake values, but the decision task expects them anyway
-                    "comment": " ",
-                },
-                hg_push_context,
-            ),
-            "now": current_json_time(),
-            "as_slugid": as_slugid,
-        },
-        context,
-    )
-
-    rendered = jsone.render(taskcluster_yml, context)
-    if len(rendered["tasks"]) != 1:
-        raise Exception("Expected .taskcluster.yml to only produce one cron task")
-    task = rendered["tasks"][0]
-
-    task_id = task.pop("taskId")
-    return (task_id, task)
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/docker.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/docker.py
index c37a69f98f..13815381ed 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/docker.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/docker.py
@@ -7,6 +7,7 @@ import hashlib
 import io
 import os
 import re
+from typing import Optional
 
 from taskgraph.util.archive import create_tar_gz_from_files
 from taskgraph.util.memoize import memoize
@@ -16,17 +17,27 @@ IMAGE_DIR = os.path.join(".", "taskcluster", "docker")
 from .yaml import load_yaml
 
 
-def docker_image(name, by_tag=False):
+def docker_image(name: str, by_tag: bool = False) -> Optional[str]:
     """
     Resolve in-tree prebuilt docker image to ``<registry>/<repository>@sha256:<digest>``,
     or ``<registry>/<repository>:<tag>`` if `by_tag` is `True`.
+
+    Args:
+        name (str): The image to build.
+        by_tag (bool): If True, will apply a tag based on VERSION file.
+            Otherwise will apply a hash based on HASH file.
+    Returns:
+        Optional[str]: Image if it can be resolved, otherwise None.
     """
     try:
         with open(os.path.join(IMAGE_DIR, name, "REGISTRY")) as f:
             registry = f.read().strip()
     except OSError:
-        with open(os.path.join(IMAGE_DIR, "REGISTRY")) as f:
-            registry = f.read().strip()
+        try:
+            with open(os.path.join(IMAGE_DIR, "REGISTRY")) as f:
+                registry = f.read().strip()
+        except OSError:
+            return None
 
     if not by_tag:
         hashfile = os.path.join(IMAGE_DIR, name, "HASH")
@@ -34,7 +45,7 @@ def docker_image(name, by_tag=False):
             with open(hashfile) as f:
                 return f"{registry}/{name}@{f.read().strip()}"
         except OSError:
-            raise Exception(f"Failed to read HASH file {hashfile}")
+            return None
 
     try:
         with open(os.path.join(IMAGE_DIR, name, "VERSION")) as f:
@@ -197,7 +208,7 @@ def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=N
 @memoize
 def image_paths():
     """Return a map of image name to paths containing their Dockerfile."""
-    config = load_yaml("taskcluster", "ci", "docker-image", "kind.yml")
+    config = load_yaml("taskcluster", "kinds", "docker-image", "kind.yml")
     return {
         k: os.path.join(IMAGE_DIR, v.get("definition", k))
         for k, v in config["tasks"].items()
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/hash.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/hash.py
index 5d884fc318..d42b2ecef9 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/hash.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/hash.py
@@ -39,10 +39,7 @@ def hash_paths(base_path, patterns):
             raise Exception("%s did not match anything" % pattern)
     for path in sorted(found):
         h.update(
-            "{} {}\n".format(
-                hash_path(mozpath.abspath(mozpath.join(base_path, path))),
-                mozpath.normsep(path),
-            ).encode("utf-8")
+            f"{hash_path(mozpath.abspath(mozpath.join(base_path, path)))} {mozpath.normsep(path)}\n".encode()
         )
     return h.hexdigest()
 
@@ -55,4 +52,8 @@ def _find_matching_files(base_path, pattern):
 
 @memoize
 def _get_all_files(base_path):
-    return [str(path) for path in Path(base_path).rglob("*") if path.is_file()]
+    return [
+        mozpath.normsep(str(path))
+        for path in Path(base_path).rglob("*")
+        if path.is_file()
+    ]
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/keyed_by.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/keyed_by.py
index 9b0c5a44fb..00c84ba980 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/keyed_by.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/keyed_by.py
@@ -66,8 +66,8 @@ def evaluate_keyed_by(
             # Error out when only 'default' is specified as only alternatives,
             # because we don't need to by-{keyed_by} there.
             raise Exception(
-                "Keyed-by '{}' unnecessary with only value 'default' "
-                "found, when determining item {}".format(keyed_by, item_name)
+                f"Keyed-by '{keyed_by}' unnecessary with only value 'default' "
+                f"found, when determining item {item_name}"
             )
 
         if key is None:
@@ -76,22 +76,20 @@ def evaluate_keyed_by(
                 continue
             else:
                 raise Exception(
-                    "No attribute {} and no value for 'default' found "
-                    "while determining item {}".format(keyed_by, item_name)
+                    f"No attribute {keyed_by} and no value for 'default' found "
+                    f"while determining item {item_name}"
                 )
 
         matches = keymatch(alternatives, key)
         if enforce_single_match and len(matches) > 1:
             raise Exception(
-                "Multiple matching values for {} {!r} found while "
-                "determining item {}".format(keyed_by, key, item_name)
+                f"Multiple matching values for {keyed_by} {key!r} found while "
+                f"determining item {item_name}"
             )
         elif matches:
             value = matches[0]
             continue
 
         raise Exception(
-            "No {} matching {!r} nor 'default' found while determining item {}".format(
-                keyed_by, key, item_name
-            )
+            f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}"
         )
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/memoize.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/memoize.py
index 56b513e74c..a4bc50cc26 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/memoize.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/memoize.py
@@ -2,39 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Imported from
-# https://searchfox.org/mozilla-central/rev/c3ebaf6de2d481c262c04bb9657eaf76bf47e2ac/python/mozbuild/mozbuild/util.py#923-949
-
-
 import functools
 
-
-class memoize(dict):
-    """A decorator to memoize the results of function calls depending
-    on its arguments.
-    Both functions and instance methods are handled, although in the
-    instance method case, the results are cache in the instance itself.
-    """
-
-    def __init__(self, func):
-        self.func = func
-        functools.update_wrapper(self, func)
-
-    def __call__(self, *args):
-        if args not in self:
-            self[args] = self.func(*args)
-        return self[args]
-
-    def method_call(self, instance, *args):
-        name = "_%s" % self.func.__name__
-        if not hasattr(instance, name):
-            setattr(instance, name, {})
-        cache = getattr(instance, name)
-        if args not in cache:
-            cache[args] = self.func(instance, *args)
-        return cache[args]
-
-    def __get__(self, instance, cls):
-        return functools.update_wrapper(
-            functools.partial(self.method_call, instance), self.func
-        )
+memoize = functools.lru_cache(maxsize=None)  # backwards compatibility shim
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/parameterization.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/parameterization.py
index 6233a98a40..1973f6f7df 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/parameterization.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/parameterization.py
@@ -20,6 +20,12 @@ def _recurse(val, param_fns):
             if len(val) == 1:
                 for param_key, param_fn in param_fns.items():
                     if set(val.keys()) == {param_key}:
+                        if isinstance(val[param_key], dict):
+                            # handle `{"task-reference": {"<foo>": "bar"}}`
+                            return {
+                                param_fn(key): recurse(v)
+                                for key, v in val[param_key].items()
+                            }
                         return param_fn(val[param_key])
             return {k: recurse(v) for k, v in val.items()}
         else:
@@ -74,17 +80,14 @@ def resolve_task_references(label, task_def, task_id, decision_task_id, dependen
                     task_id = dependencies[dependency]
                 except KeyError:
                     raise KeyError(
-                        "task '{}' has no dependency named '{}'".format(
-                            label, dependency
-                        )
+                        f"task '{label}' has no dependency named '{dependency}'"
                     )
 
-            assert artifact_name.startswith(
-                "public/"
-            ), "artifact-reference only supports public artifacts, not `{}`".format(
-                artifact_name
-            )
-            return get_artifact_url(task_id, artifact_name)
+            use_proxy = False
+            if not artifact_name.startswith("public/"):
+                use_proxy = True
+
+            return get_artifact_url(task_id, artifact_name, use_proxy=use_proxy)
 
         return ARTIFACT_REFERENCE_PATTERN.sub(repl, val)
 
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/schema.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/schema.py
index 3989f71182..02e79a3a27 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/schema.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/schema.py
@@ -74,7 +74,7 @@ def resolve_keyed_by(
 
     For example, given item::
 
-        job:
+        task:
             test-platform: linux128
             chunks:
                 by-test-platform:
@@ -82,10 +82,10 @@ def resolve_keyed_by(
                     win.*: 6
                     default: 12
 
-    a call to `resolve_keyed_by(item, 'job.chunks', item['thing-name'])`
+    a call to `resolve_keyed_by(item, 'task.chunks', item['thing-name'])`
     would mutate item in-place to::
 
-        job:
+        task:
             test-platform: linux128
             chunks: 12
 
@@ -182,7 +182,7 @@ def check_schema(schema):
                 if not identifier_re.match(k) and not excepted(path):
                     raise RuntimeError(
                         "YAML schemas should use dashed lower-case identifiers, "
-                        "not {!r} @ {}".format(k, path)
+                        f"not {k!r} @ {path}"
                     )
             elif isinstance(k, (voluptuous.Optional, voluptuous.Required)):
                 check_identifier(path, k.schema)
@@ -191,9 +191,7 @@ def check_schema(schema):
                     check_identifier(path, v)
             elif not excepted(path):
                 raise RuntimeError(
-                    "Unexpected type in YAML schema: {} @ {}".format(
-                        type(k).__name__, path
-                    )
+                    f"Unexpected type in YAML schema: {type(k).__name__} @ {path}"
                 )
 
         if isinstance(sch, collections.abc.Mapping):
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/set_name.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/set_name.py
new file mode 100644
index 0000000000..4c27a9cca1
--- /dev/null
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/set_name.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Define a collection of set_name functions
+# Note: this is stored here instead of where it is used in the `from_deps`
+# transform to give consumers a chance to register their own `set_name`
+# handlers before the `from_deps` schema is created.
+SET_NAME_MAP = {}
+
+
+def set_name(name, schema=None):
+    def wrapper(func):
+        assert (
+            name not in SET_NAME_MAP
+        ), f"duplicate set_name function name {name} ({func} and {SET_NAME_MAP[name]})"
+        SET_NAME_MAP[name] = func
+        func.schema = schema
+        return func
+
+    return wrapper
+
+
+@set_name("strip-kind")
+def set_name_strip_kind(config, tasks, primary_dep, primary_kind):
+    if primary_dep.label.startswith(primary_kind):
+        return primary_dep.label[len(primary_kind) + 1 :]
+    else:
+        return primary_dep.label
+
+
+@set_name("retain-kind")
+def set_name_retain_kind(config, tasks, primary_dep, primary_kind):
+    return primary_dep.label
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/shell.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/shell.py
index d695767f05..16b71b7d6a 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/shell.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/shell.py
@@ -14,7 +14,7 @@ def _quote(s):
     As a special case, if given an int, returns a string containing the int,
     not enclosed in quotes.
     """
-    if type(s) == int:
+    if isinstance(s, int):
         return "%d" % s
 
     # Empty strings need to be quoted to have any significance
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/taskcluster.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/taskcluster.py
index a830a473b3..b467e98a97 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/taskcluster.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/taskcluster.py
@@ -3,10 +3,12 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import copy
 import datetime
 import functools
 import logging
 import os
+from typing import Dict, List, Union
 
 import requests
 import taskcluster_urls as liburls
@@ -53,9 +55,11 @@ def get_root_url(use_proxy):
         logger.debug(
             "Running in Taskcluster instance {}{}".format(
                 os.environ["TASKCLUSTER_ROOT_URL"],
-                " with taskcluster-proxy"
-                if "TASKCLUSTER_PROXY_URL" in os.environ
-                else "",
+                (
+                    " with taskcluster-proxy"
+                    if "TASKCLUSTER_PROXY_URL" in os.environ
+                    else ""
+                ),
             )
         )
         return liburls.normalize_root_url(os.environ["TASKCLUSTER_ROOT_URL"])
@@ -136,22 +140,9 @@ def _handle_artifact(path, response):
 
 def get_artifact_url(task_id, path, use_proxy=False):
     artifact_tmpl = liburls.api(
-        get_root_url(False), "queue", "v1", "task/{}/artifacts/{}"
+        get_root_url(use_proxy), "queue", "v1", "task/{}/artifacts/{}"
     )
-    data = artifact_tmpl.format(task_id, path)
-    if use_proxy:
-        # Until Bug 1405889 is deployed, we can't download directly
-        # from the taskcluster-proxy.  Work around by using the /bewit
-        # endpoint instead.
-        # The bewit URL is the body of a 303 redirect, which we don't
-        # want to follow (which fetches a potentially large resource).
-        response = _do_request(
-            os.environ["TASKCLUSTER_PROXY_URL"] + "/bewit",
-            data=data,
-            allow_redirects=False,
-        )
-        return response.text
-    return data
+    return artifact_tmpl.format(task_id, path)
 
 
 def get_artifact(task_id, path, use_proxy=False):
@@ -244,6 +235,7 @@ def get_task_url(task_id, use_proxy=False):
     return task_tmpl.format(task_id)
 
 
+@memoize
 def get_task_definition(task_id, use_proxy=False):
     response = _do_request(get_task_url(task_id, use_proxy))
     return response.json()
@@ -327,11 +319,7 @@ def get_purge_cache_url(provisioner_id, worker_type, use_proxy=False):
 def purge_cache(provisioner_id, worker_type, cache_name, use_proxy=False):
     """Requests a cache purge from the purge-caches service."""
     if testing:
-        logger.info(
-            "Would have purged {}/{}/{}.".format(
-                provisioner_id, worker_type, cache_name
-            )
-        )
+        logger.info(f"Would have purged {provisioner_id}/{worker_type}/{cache_name}.")
     else:
         logger.info(f"Purging {provisioner_id}/{worker_type}/{cache_name}.")
         purge_cache_url = get_purge_cache_url(provisioner_id, worker_type, use_proxy)
@@ -371,3 +359,40 @@ def list_task_group_incomplete_tasks(task_group_id):
             params = {"continuationToken": resp.get("continuationToken")}
         else:
             break
+
+
+@memoize
+def _get_deps(task_ids, use_proxy):
+    upstream_tasks = {}
+    for task_id in task_ids:
+        task_def = get_task_definition(task_id, use_proxy)
+        upstream_tasks[task_def["metadata"]["name"]] = task_id
+
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
+
+    return upstream_tasks
+
+
+def get_ancestors(
+    task_ids: Union[List[str], str], use_proxy: bool = False
+) -> Dict[str, str]:
+    """Gets the ancestor tasks of the given task_ids as a dictionary of label -> taskid.
+
+    Args:
+        task_ids (str or [str]): A single task id or a list of task ids to find the ancestors of.
+        use_proxy (bool): See get_root_url.
+
+    Returns:
+        dict: A dict whose keys are task labels and values are task ids.
+    """
+    upstream_tasks: Dict[str, str] = {}
+
+    if isinstance(task_ids, str):
+        task_ids = [task_ids]
+
+    for task_id in task_ids:
+        task_def = get_task_definition(task_id, use_proxy)
+
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
+
+    return copy.deepcopy(upstream_tasks)
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/time.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/time.py
index e511978b5f..6639e5dddd 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/time.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/time.py
@@ -73,9 +73,7 @@ def value_of(input_str):
 
     if unit not in ALIASES:
         raise UnknownTimeMeasurement(
-            "{} is not a valid time measure use one of {}".format(
-                unit, sorted(ALIASES.keys())
-            )
+            f"{unit} is not a valid time measure use one of {sorted(ALIASES.keys())}"
         )
 
     return ALIASES[unit](value)
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/treeherder.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/treeherder.py
index cff5f286cc..6bb6dbd137 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/treeherder.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/treeherder.py
@@ -42,22 +42,25 @@ def replace_group(treeherder_symbol, new_group):
     return join_symbol(new_group, symbol)
 
 
-def inherit_treeherder_from_dep(job, dep_job):
-    """Inherit treeherder defaults from dep_job"""
-    treeherder = job.get("treeherder", {})
+def inherit_treeherder_from_dep(task, dep_task):
+    """Inherit treeherder defaults from dep_task"""
+    treeherder = task.get("treeherder", {})
 
     dep_th_platform = (
-        dep_job.task.get("extra", {})
+        dep_task.task.get("extra", {})
         .get("treeherder", {})
         .get("machine", {})
         .get("platform", "")
     )
     dep_th_collection = list(
-        dep_job.task.get("extra", {}).get("treeherder", {}).get("collection", {}).keys()
+        dep_task.task.get("extra", {})
+        .get("treeherder", {})
+        .get("collection", {})
+        .keys()
     )[0]
     treeherder.setdefault("platform", f"{dep_th_platform}/{dep_th_collection}")
     treeherder.setdefault(
-        "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
+        "tier", dep_task.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
     )
     # Does not set symbol
     treeherder.setdefault("kind", "build")
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py
index 2d967d2645..c2fd0d3236 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py
@@ -10,9 +10,6 @@ import subprocess
 from abc import ABC, abstractmethod, abstractproperty
 from shutil import which
 
-import requests
-from redo import retry
-
 from taskgraph.util.path import ancestors
 
 PUSHLOG_TMPL = "{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1"
@@ -21,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 
 class Repository(ABC):
-    # Both mercurial and git use sha1 as revision idenfiers. Luckily, both define
+    # Both mercurial and git use sha1 as revision identifiers. Luckily, both define
     # the same value as the null revision.
     #
     # https://github.com/git/git/blob/dc04167d378fb29d30e1647ff6ff51dd182bc9a3/t/oid-info/hash-info#L7
@@ -519,34 +516,3 @@ def get_repository(path):
             return GitRepository(path)
 
     raise RuntimeError("Current directory is neither a git or hg repository")
-
-
-def find_hg_revision_push_info(repository, revision):
-    """Given the parameters for this action and a revision, find the
-    pushlog_id of the revision."""
-    pushlog_url = PUSHLOG_TMPL.format(repository, revision)
-
-    def query_pushlog(url):
-        r = requests.get(pushlog_url, timeout=60)
-        r.raise_for_status()
-        return r
-
-    r = retry(
-        query_pushlog,
-        args=(pushlog_url,),
-        attempts=5,
-        sleeptime=10,
-    )
-    pushes = r.json()["pushes"]
-    if len(pushes) != 1:
-        raise RuntimeError(
-            "Unable to find a single pushlog_id for {} revision {}: {}".format(
-                repository, revision, pushes
-            )
-        )
-    pushid = list(pushes.keys())[0]
-    return {
-        "pushdate": pushes[pushid]["date"],
-        "pushid": pushid,
-        "user": pushes[pushid]["user"],
-    }
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py
index e6705c16cf..b5bb0889ae 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py
@@ -134,10 +134,8 @@ def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, paramet
             collection_keys = tuple(sorted(treeherder.get("collection", {}).keys()))
             if len(collection_keys) != 1:
                 raise Exception(
-                    "Task {} can't be in multiple treeherder collections "
-                    "(the part of the platform after `/`): {}".format(
-                        task.label, collection_keys
-                    )
+                    f"Task {task.label} can't be in multiple treeherder collections "
+                    f"(the part of the platform after `/`): {collection_keys}"
                 )
             platform = treeherder.get("machine", {}).get("platform")
             group_symbol = treeherder.get("groupSymbol")
@@ -175,9 +173,7 @@ def verify_trust_domain_v2_routes(
         if route.startswith(route_prefix):
             if route in scratch_pad:
                 raise Exception(
-                    "conflict between {}:{} for route: {}".format(
-                        task.label, scratch_pad[route], route
-                    )
+                    f"conflict between {task.label}:{scratch_pad[route]} for route: {route}"
                 )
             else:
                 scratch_pad[route] = task.label
@@ -206,9 +202,7 @@ def verify_routes_notification_filters(
             route_filter = route.split(".")[-1]
             if route_filter not in valid_filters:
                 raise Exception(
-                    "{} has invalid notification filter ({})".format(
-                        task.label, route_filter
-                    )
+                    f"{task.label} has invalid notification filter ({route_filter})"
                 )
 
 
@@ -235,12 +229,7 @@ def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, paramete
                     continue
                 if tier < tiers[d]:
                     raise Exception(
-                        "{} (tier {}) cannot depend on {} (tier {})".format(
-                            task.label,
-                            printable_tier(tier),
-                            d,
-                            printable_tier(tiers[d]),
-                        )
+                        f"{task.label} (tier {printable_tier(tier)}) cannot depend on {d} (tier {printable_tier(tiers[d])})"
                     )
 
 
@@ -262,11 +251,7 @@ def verify_toolchain_alias(task, taskgraph, scratch_pad, graph_config, parameter
             if key in scratch_pad:
                 raise Exception(
                     "Duplicate toolchain-alias in tasks "
-                    "`{}`and `{}`: {}".format(
-                        task.label,
-                        scratch_pad[key],
-                        key,
-                    )
+                    f"`{task.label}`and `{scratch_pad[key]}`: {key}"
                 )
             else:
                 scratch_pad[key] = task.label
diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/yaml.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/yaml.py
index 141c7a16d3..a733521527 100644
--- a/third_party/python/taskcluster_taskgraph/taskgraph/util/yaml.py
+++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/yaml.py
@@ -5,7 +5,10 @@
 
 import os
 
-from yaml.loader import SafeLoader
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
 
 
 class UnicodeLoader(SafeLoader):