110 lines
3.7 KiB
Python
110 lines
3.7 KiB
Python
# mypy: allow-untyped-defs
|
|
|
|
import argparse
|
|
import os
|
|
import logging
|
|
|
|
import requests
|
|
|
|
import github
|
|
|
|
|
|
logging.basicConfig()
|
|
logger = logging.getLogger("tc-download")
|
|
|
|
# The root URL of the Taskcluster deployment from which to download wpt reports
|
|
# (after https://bugzilla.mozilla.org/show_bug.cgi?id=1574668 lands, this will
|
|
# be https://community-tc.services.mozilla.com)
|
|
TASKCLUSTER_ROOT_URL = 'https://taskcluster.net'
|
|
|
|
|
|
def get_parser():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--ref", default="master",
|
|
help="Branch (in the GitHub repository) or commit to fetch logs for")
|
|
parser.add_argument("--artifact-name", default="wpt_report.json.gz",
|
|
help="Log type to fetch")
|
|
parser.add_argument("--repo-name", default="web-platform-tests/wpt",
|
|
help="GitHub repo name in the format owner/repo. "
|
|
"This must be the repo from which the Taskcluster run was scheduled "
|
|
"(for PRs this is the repo into which the PR would merge)")
|
|
parser.add_argument("--token-file", help="File containing GitHub token")
|
|
parser.add_argument("--out-dir", default=".",
|
|
help="Path to save the logfiles")
|
|
return parser
|
|
|
|
|
|
def get_json(url, key=None):
|
|
resp = requests.get(url)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
if key:
|
|
data = data[key]
|
|
return data
|
|
|
|
|
|
def get(url, dest, name):
|
|
resp = requests.get(url)
|
|
resp.raise_for_status()
|
|
path = os.path.join(dest, name)
|
|
with open(path, "w") as f:
|
|
f.write(resp.content)
|
|
return path
|
|
|
|
|
|
def run(*args, **kwargs):
|
|
if not os.path.exists(kwargs["out_dir"]):
|
|
os.mkdir(kwargs["out_dir"])
|
|
|
|
if kwargs["token_file"]:
|
|
with open(kwargs["token_file"]) as f:
|
|
gh = github.Github(f.read().strip())
|
|
else:
|
|
gh = github.Github()
|
|
|
|
repo = gh.get_repo(kwargs["repo_name"])
|
|
commit = repo.get_commit(kwargs["ref"])
|
|
statuses = commit.get_statuses()
|
|
taskgroups = set()
|
|
|
|
for status in statuses:
|
|
if not status.context.startswith("Taskcluster "):
|
|
continue
|
|
if status.state == "pending":
|
|
continue
|
|
taskgroup_id = status.target_url.rsplit("/", 1)[1]
|
|
taskgroups.add(taskgroup_id)
|
|
|
|
if not taskgroups:
|
|
logger.error("No complete Taskcluster runs found for ref %s" % kwargs["ref"])
|
|
return 1
|
|
|
|
for taskgroup in taskgroups:
|
|
if TASKCLUSTER_ROOT_URL == 'https://taskcluster.net':
|
|
# NOTE: this condition can be removed after November 9, 2019
|
|
taskgroup_url = "https://queue.taskcluster.net/v1/task-group/%s/list"
|
|
artifacts_list_url = "https://queue.taskcluster.net/v1/task/%s/artifacts"
|
|
else:
|
|
taskgroup_url = TASKCLUSTER_ROOT_URL + "/api/queue/v1/task-group/%s/list"
|
|
artifacts_list_url = TASKCLUSTER_ROOT_URL + "/api/queue/v1/task/%s/artifacts"
|
|
tasks = get_json(taskgroup_url % taskgroup, "tasks")
|
|
for task in tasks:
|
|
task_id = task["status"]["taskId"]
|
|
url = artifacts_list_url % (task_id,)
|
|
for artifact in get_json(url, "artifacts"):
|
|
if artifact["name"].endswith(kwargs["artifact_name"]):
|
|
filename = "%s-%s-%s" % (task["task"]["metadata"]["name"],
|
|
task_id,
|
|
kwargs["artifact_name"])
|
|
path = get("%s/%s" % (url, artifact["name"]), kwargs["out_dir"], filename)
|
|
logger.info(path)
|
|
|
|
|
|
def main():
|
|
kwargs = get_parser().parse_args()
|
|
|
|
run(None, vars(kwargs))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() # type: ignore
|