summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/update
diff options
context:
space:
mode:
Diffstat (limited to 'testing/web-platform/update')
-rw-r--r--testing/web-platform/update/__init__.py40
-rw-r--r--testing/web-platform/update/fetchlogs.py123
-rw-r--r--testing/web-platform/update/github.py169
-rw-r--r--testing/web-platform/update/tree.py196
-rw-r--r--testing/web-platform/update/update.py51
-rw-r--r--testing/web-platform/update/updatecommandline.py58
-rw-r--r--testing/web-platform/update/upstream.py475
7 files changed, 1112 insertions, 0 deletions
diff --git a/testing/web-platform/update/__init__.py b/testing/web-platform/update/__init__.py
new file mode 100644
index 0000000000..47c6d36d95
--- /dev/null
+++ b/testing/web-platform/update/__init__.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import imp
+import os
+import sys
+
+from mozlog import structuredlog
+
+here = os.path.split(__file__)[0]
+
+imp.load_source(
+ "localpaths", os.path.join(here, os.pardir, "tests", "tools", "localpaths.py")
+)
+
+from wptrunner.update import WPTUpdate, setup_logging
+from wptrunner.update.base import exit_unclean
+
+from . import updatecommandline
+from .update import UpdateRunner
+
+
+def run_update(logger, **kwargs):
+ updater = WPTUpdate(logger, runner_cls=UpdateRunner, **kwargs)
+ return updater.run()
+
+
+if __name__ == "__main__":
+ args = updatecommandline.parse_args()
+ logger = setup_logging(args, {"mach": sys.stdout})
+ assert structuredlog.get_default_logger() is not None
+
+ rv = run_update(logger, **args)
+ if rv is exit_unclean:
+ sys.exit(1)
+ else:
+ sys.exit(0)
diff --git a/testing/web-platform/update/fetchlogs.py b/testing/web-platform/update/fetchlogs.py
new file mode 100644
index 0000000000..dc8ebae059
--- /dev/null
+++ b/testing/web-platform/update/fetchlogs.py
@@ -0,0 +1,123 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+
+import requests
+import urlparse
+
+treeherder_base = "https://treeherder.mozilla.org/"
+
+"""Simple script for downloading structured logs from treeherder.
+
+For the moment this is specialised to work with web-platform-tests
+logs; in due course it should move somewhere generic and get hooked
+up to mach or similar"""
+
+# Interpretation of the "job" list from
+# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18
+
+
+def create_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("branch", action="store", help="Branch on which jobs ran")
+ parser.add_argument("commit", action="store", help="Commit hash for push")
+
+ return parser
+
+
+def download(url, prefix, dest, force_suffix=True):
+ if dest is None:
+ dest = "."
+
+ if prefix and not force_suffix:
+ name = os.path.join(dest, prefix + ".log")
+ else:
+ name = None
+ counter = 0
+
+ while not name or os.path.exists(name):
+ counter += 1
+ sep = "" if not prefix else "-"
+ name = os.path.join(dest, prefix + sep + str(counter) + ".log")
+
+ with open(name, "wb") as f:
+ resp = requests.get(url, stream=True)
+ for chunk in resp.iter_content(1024):
+ f.write(chunk)
+
+
+def fetch_json(url, params=None):
+ headers = {
+ "Accept": "application/json",
+ "User-Agent": "wpt-fetchlogs",
+ }
+ response = requests.get(url=url, params=params, headers=headers, timeout=30)
+ response.raise_for_status()
+ return response.json()
+
+
+def get_blobber_url(branch, job):
+ job_guid = job["job_guid"]
+ artifact_url = urlparse.urljoin(treeherder_base, "/api/jobdetail/")
+ artifact_params = {
+ "job_guid": job_guid,
+ }
+ job_data = fetch_json(artifact_url, params=artifact_params)
+
+ if job_data:
+ try:
+ for item in job_data["results"]:
+ if item["value"] == "wpt_raw.log" or item["value"] == "log_raw.log":
+ return item["url"]
+ except Exception:
+ return None
+
+
+def get_structured_logs(branch, commit, dest=None):
+ resultset_url = urlparse.urljoin(
+ treeherder_base, "/api/project/%s/resultset/" % branch
+ )
+ resultset_params = {
+ "revision": commit,
+ }
+ revision_data = fetch_json(resultset_url, params=resultset_params)
+ result_set = revision_data["results"][0]["id"]
+
+ jobs_url = urlparse.urljoin(treeherder_base, "/api/project/%s/jobs/" % branch)
+ jobs_params = {
+ "result_set_id": result_set,
+ "count": 2000,
+ "exclusion_profile": "false",
+ }
+ job_data = fetch_json(jobs_url, params=jobs_params)
+
+ tasks = []
+
+ for result in job_data["results"]:
+ job_type_name = result["job_type_name"]
+ if (
+ job_type_name.startswith("W3C Web Platform")
+ or job_type_name.startswith("test-")
+ and "-web-platform-tests-" in job_type_name
+ ):
+ url = get_blobber_url(branch, result)
+ if url:
+ prefix = result["platform"] # platform
+ tasks.append((url, prefix, None))
+
+ for task in tasks:
+ download(*task)
+
+
+def main():
+ parser = create_parser()
+ args = parser.parse_args()
+
+ get_structured_logs(args.branch, args.commit)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/testing/web-platform/update/github.py b/testing/web-platform/update/github.py
new file mode 100644
index 0000000000..ba5eb2234a
--- /dev/null
+++ b/testing/web-platform/update/github.py
@@ -0,0 +1,169 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from six.moves.urllib.parse import urljoin
+
+requests = None
+
+
+class GitHubError(Exception):
+ def __init__(self, status, data):
+ self.status = status
+ self.data = data
+
+
+class GitHub(object):
+ url_base = "https://api.github.com"
+
+ def __init__(self, token):
+ # Defer the import of requests since it isn't installed by default
+ global requests
+ if requests is None:
+ import requests
+
+ self.headers = {"Accept": "application/vnd.github.v3+json"}
+ self.auth = (token, "x-oauth-basic")
+
+ def get(self, path):
+ return self._request("GET", path)
+
+ def post(self, path, data):
+ return self._request("POST", path, data=data)
+
+ def put(self, path, data, headers=None):
+ return self._request("PUT", path, data=data, headers=headers)
+
+ def _request(self, method, path, data=None, headers=None):
+ url = urljoin(self.url_base, path)
+
+ headers_ = self.headers
+ if headers is not None:
+ headers_.update(headers)
+ kwargs = {"headers": headers_, "auth": self.auth}
+ if data is not None:
+ kwargs["data"] = json.dumps(data)
+
+ resp = requests.request(method, url, **kwargs)
+
+ if 200 <= resp.status_code < 300:
+ return resp.json()
+ else:
+ print(method, path, resp.status_code, resp.json())
+ raise GitHubError(resp.status_code, resp.json())
+
+ def repo(self, owner, name):
+ """GitHubRepo for a particular repository.
+
+ :param owner: String repository owner
+ :param name: String repository name
+ """
+ return GitHubRepo.from_name(self, owner, name)
+
+
+class GitHubRepo(object):
+ def __init__(self, github, data):
+ """Object representing a GitHub repository"""
+ self.gh = github
+ self.owner = data["owner"]
+ self.name = data["name"]
+ self.url = data["ssh_url"]
+ self._data = data
+
+ @classmethod
+ def from_name(cls, github, owner, name):
+ data = github.get("/repos/%s/%s" % (owner, name))
+ return cls(github, data)
+
+ @property
+ def url_base(self):
+ return "/repos/%s/" % (self._data["full_name"])
+
+ def create_pr(self, title, head, base, body):
+ """Create a Pull Request in the repository
+
+ :param title: Pull Request title
+ :param head: ref to the HEAD of the PR branch.
+ :param base: ref to the base branch for the Pull Request
+ :param body: Description of the PR
+ """
+ return PullRequest.create(self, title, head, base, body)
+
+ def load_pr(self, number):
+ """Load an existing Pull Request by number.
+
+ :param number: Pull Request number
+ """
+ return PullRequest.from_number(self, number)
+
+ def path(self, suffix):
+ return urljoin(self.url_base, suffix)
+
+
+class PullRequest(object):
+ def __init__(self, repo, data):
+ """Object representing a Pull Request"""
+
+ self.repo = repo
+ self._data = data
+ self.number = data["number"]
+ self.title = data["title"]
+ self.base = data["base"]["ref"]
+ self.base = data["head"]["ref"]
+ self._issue = None
+
+ @classmethod
+ def from_number(cls, repo, number):
+ data = repo.gh.get(repo.path("pulls/%i" % number))
+ return cls(repo, data)
+
+ @classmethod
+ def create(cls, repo, title, head, base, body):
+ data = repo.gh.post(
+ repo.path("pulls"),
+ {"title": title, "head": head, "base": base, "body": body},
+ )
+ return cls(repo, data)
+
+ def path(self, suffix):
+ return urljoin(self.repo.path("pulls/%i/" % self.number), suffix)
+
+ @property
+ def issue(self):
+ """Issue related to the Pull Request"""
+ if self._issue is None:
+ self._issue = Issue.from_number(self.repo, self.number)
+ return self._issue
+
+ def merge(self):
+ """Merge the Pull Request into its base branch."""
+ self.repo.gh.put(
+ self.path("merge"),
+ {"merge_method": "merge"},
+ headers={"Accept": "application/vnd.github.polaris-preview+json"},
+ )
+
+
+class Issue(object):
+ def __init__(self, repo, data):
+ """Object representing a GitHub Issue"""
+ self.repo = repo
+ self._data = data
+ self.number = data["number"]
+
+ @classmethod
+ def from_number(cls, repo, number):
+ data = repo.gh.get(repo.path("issues/%i" % number))
+ return cls(repo, data)
+
+ def path(self, suffix):
+ return urljoin(self.repo.path("issues/%i/" % self.number), suffix)
+
+ def add_comment(self, message):
+ """Add a comment to the issue
+
+ :param message: The text of the comment
+ """
+ self.repo.gh.post(self.path("comments"), {"body": message})
diff --git a/testing/web-platform/update/tree.py b/testing/web-platform/update/tree.py
new file mode 100644
index 0000000000..48197c74b6
--- /dev/null
+++ b/testing/web-platform/update/tree.py
@@ -0,0 +1,196 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import tempfile
+
+from wptrunner import update as wptupdate
+from wptrunner.update.tree import Commit, CommitMessage, get_unique_name
+
+
+class HgTree(wptupdate.tree.HgTree):
+ def __init__(self, *args, **kwargs):
+ self.commit_cls = kwargs.pop("commit_cls", Commit)
+ wptupdate.tree.HgTree.__init__(self, *args, **kwargs)
+
+ # TODO: The extra methods for upstreaming patches from a
+ # hg checkout
+
+
+class GitTree(wptupdate.tree.GitTree):
+ def __init__(self, *args, **kwargs):
+ """Extension of the basic GitTree with extra methods for
+ transfering patches"""
+ commit_cls = kwargs.pop("commit_cls", Commit)
+ wptupdate.tree.GitTree.__init__(self, *args, **kwargs)
+ self.commit_cls = commit_cls
+
+ def rev_from_hg(self, rev):
+ return self.git("cinnabar", "hg2git", rev).strip()
+
+ def rev_to_hg(self, rev):
+ return self.git("cinnabar", "git2hg", rev).strip()
+
+ def create_branch(self, name, ref=None):
+ """Create a named branch,
+
+ :param name: String representing the branch name.
+ :param ref: None to use current HEAD or rev that the branch should point to"""
+
+ args = []
+ if ref is not None:
+ if hasattr(ref, "sha1"):
+ ref = ref.sha1
+ args.append(ref)
+ self.git("branch", name, *args)
+
+ def commits_by_message(self, message, path=None):
+ """List of commits with messages containing a given string.
+
+ :param message: The string that must be contained in the message.
+ :param path: Path to a file or directory the commit touches
+ """
+ args = ["--pretty=format:%H", "--reverse", "-z", "--grep=%s" % message]
+ if path is not None:
+ args.append("--")
+ args.append(path)
+ data = self.git("log", *args)
+ return [self.commit_cls(self, sha1) for sha1 in data.split("\0")]
+
+ def log(self, base_commit=None, path=None):
+ """List commits touching a certian path from a given base commit.
+
+ :base_param commit: Commit object for the base commit from which to log
+ :param path: Path that the commits must touch
+ """
+ args = ["--pretty=format:%H", "--reverse", "-z"]
+ if base_commit is not None:
+ args.append("%s.." % base_commit.sha1)
+ if path is not None:
+ args.append("--")
+ args.append(path)
+ data = self.git("log", *args)
+ return [self.commit_cls(self, sha1) for sha1 in data.split("\0") if sha1]
+
+ def import_patch(self, patch):
+ """Import a patch file into the tree and commit it
+
+ :param patch: a Patch object containing the patch to import
+ """
+
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(patch.diff)
+ f.flush()
+ f.seek(0)
+ self.git("apply", "--index", f.name)
+ self.git("commit", "-m", patch.message.text, "--author=%s" % patch.full_author)
+
+ def rebase(self, ref, continue_rebase=False):
+ """Rebase the current branch onto another commit.
+
+ :param ref: A Commit object for the commit to rebase onto
+ :param continue_rebase: Continue an in-progress rebase"""
+ if continue_rebase:
+ args = ["--continue"]
+ else:
+ if hasattr(ref, "sha1"):
+ ref = ref.sha1
+ args = [ref]
+ self.git("rebase", *args)
+
+ def push(self, remote, local_ref, remote_ref, force=False):
+ """Push local changes to a remote.
+
+ :param remote: URL of the remote to push to
+ :param local_ref: Local branch to push
+ :param remote_ref: Name of the remote branch to push to
+ :param force: Do a force push
+ """
+ args = []
+ if force:
+ args.append("-f")
+ args.extend([remote, "%s:%s" % (local_ref, remote_ref)])
+ self.git("push", *args)
+
+ def unique_branch_name(self, prefix):
+ """Get an unused branch name in the local tree
+
+ :param prefix: Prefix to use at the start of the branch name"""
+ branches = [
+ ref[len("refs/heads/") :]
+ for sha1, ref in self.list_refs()
+ if ref.startswith("refs/heads/")
+ ]
+ return get_unique_name(branches, prefix)
+
+
+class Patch(object):
+ def __init__(self, author, email, message, diff):
+ self.author = author
+ self.email = email
+ if isinstance(message, CommitMessage):
+ self.message = message
+ else:
+ self.message = GeckoCommitMessage(message)
+ self.diff = diff
+
+ def __repr__(self):
+ return "<Patch (%s)>" % self.message.full_summary
+
+ @property
+ def full_author(self):
+ return "%s <%s>" % (self.author, self.email)
+
+ @property
+ def empty(self):
+ return bool(self.diff.strip())
+
+
+class GeckoCommitMessage(CommitMessage):
+ """Commit message following the Gecko conventions for identifying bug number
+ and reviewer"""
+
+ # c.f. http://hg.mozilla.org/hgcustom/version-control-tools/file/tip/hghooks/mozhghooks/commit-message.py # noqa E501
+ # which has the regexps that are actually enforced by the VCS hooks. These are
+ # slightly different because we need to parse out specific parts of the message rather
+ # than just enforce a general pattern.
+
+ _bug_re = re.compile(
+ "^Bug (\d+)[^\w]*(?:Part \d+[^\w]*)?(.*?)\s*(?:r=(\w*))?$", re.IGNORECASE
+ )
+
+ _backout_re = re.compile(
+ "^(?:Back(?:ing|ed)\s+out)|Backout|(?:Revert|(?:ed|ing))", re.IGNORECASE
+ )
+ _backout_sha1_re = re.compile("(?:\s|\:)(0-9a-f){12}")
+
+ def _parse_message(self):
+ CommitMessage._parse_message(self)
+
+ if self._backout_re.match(self.full_summary):
+ self.backouts = self._backout_re.findall(self.full_summary)
+ else:
+ self.backouts = []
+
+ m = self._bug_re.match(self.full_summary)
+ if m is not None:
+ self.bug, self.summary, self.reviewer = m.groups()
+ else:
+ self.bug, self.summary, self.reviewer = None, self.full_summary, None
+
+
+class GeckoCommit(Commit):
+ msg_cls = GeckoCommitMessage
+
+ def export_patch(self, path=None):
+ """Convert a commit in the tree to a Patch with the bug number and
+ reviewer stripped from the message"""
+ args = ["--binary", "--patch", "--format=format:", "%s" % (self.sha1,)]
+ if path is not None:
+ args.append("--")
+ args.append(path)
+
+ diff = self.git("show", *args)
+
+ return Patch(self.author, self.email, self.message, diff)
diff --git a/testing/web-platform/update/update.py b/testing/web-platform/update/update.py
new file mode 100644
index 0000000000..68d3293fb7
--- /dev/null
+++ b/testing/web-platform/update/update.py
@@ -0,0 +1,51 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+from wptrunner.update.base import Step, StepRunner
+from wptrunner.update.tree import NoVCSTree
+from wptrunner.update.update import (
+ LoadConfig,
+ RemoveObsolete,
+ SyncFromUpstream,
+ UpdateMetadata,
+)
+
+from .tree import GeckoCommit, GitTree, HgTree
+from .upstream import SyncToUpstream
+
+
+class LoadTrees(Step):
+ """Load gecko tree and sync tree containing web-platform-tests"""
+
+ provides = ["local_tree", "sync_tree"]
+
+ def create(self, state):
+ if os.path.exists(state.sync["path"]):
+ sync_tree = GitTree(root=state.sync["path"])
+ else:
+ sync_tree = None
+
+ if GitTree.is_type():
+ local_tree = GitTree(commit_cls=GeckoCommit)
+ elif HgTree.is_type():
+ local_tree = HgTree(commit_cls=GeckoCommit)
+ else:
+ local_tree = NoVCSTree()
+
+ state.update({"local_tree": local_tree, "sync_tree": sync_tree})
+
+
+class UpdateRunner(StepRunner):
+ """Overall runner for updating web-platform-tests in Gecko."""
+
+ steps = [
+ LoadConfig,
+ LoadTrees,
+ SyncToUpstream,
+ SyncFromUpstream,
+ RemoveObsolete,
+ UpdateMetadata,
+ ]
diff --git a/testing/web-platform/update/updatecommandline.py b/testing/web-platform/update/updatecommandline.py
new file mode 100644
index 0000000000..9e8cd8673e
--- /dev/null
+++ b/testing/web-platform/update/updatecommandline.py
@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def create_parser():
+ from wptrunner import wptcommandline
+
+ parser = wptcommandline.create_parser_update()
+ parser.add_argument(
+ "--upstream",
+ dest="upstream",
+ action="store_true",
+ default=None,
+ help="Push local changes to upstream repository even when not syncing",
+ )
+ parser.add_argument(
+ "--no-upstream",
+ dest="upstream",
+ action="store_false",
+ default=None,
+ help="Dont't push local changes to upstream repository when syncing",
+ )
+ parser.add_argument(
+ "--token-file",
+ action="store",
+ type=wptcommandline.abs_path,
+ help="Path to file containing github token",
+ )
+ parser.add_argument("--token", action="store", help="GitHub token to use")
+ return parser
+
+
+def check_args(kwargs):
+ from wptrunner import wptcommandline
+
+ kwargs = wptcommandline.check_args_update(kwargs)
+ kwargs["upstream"] = (
+ kwargs["upstream"] if kwargs["upstream"] is not None else kwargs["sync"]
+ )
+
+ if kwargs["upstream"]:
+ if kwargs["rev"]:
+ raise ValueError("Setting --rev with --upstream isn't supported")
+ if kwargs["token"] is None:
+ if kwargs["token_file"] is None:
+ raise ValueError("Must supply either a token file or a token")
+ with open(kwargs["token_file"]) as f:
+ token = f.read().strip()
+ kwargs["token"] = token
+ del kwargs["token_file"]
+ return kwargs
+
+
+def parse_args():
+ parser = create_parser()
+ kwargs = vars(parser.parse_args())
+ return check_args(kwargs)
diff --git a/testing/web-platform/update/upstream.py b/testing/web-platform/update/upstream.py
new file mode 100644
index 0000000000..21efc72a13
--- /dev/null
+++ b/testing/web-platform/update/upstream.py
@@ -0,0 +1,475 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+from six.moves import input
+from six.moves.urllib import parse as urlparse
+from wptrunner.update.base import Step, StepRunner, exit_clean, exit_unclean
+from wptrunner.update.tree import get_unique_name
+
+from .github import GitHub
+from .tree import Commit, GitTree, Patch
+
+
+def rewrite_patch(patch, strip_dir):
+ """Take a Patch and convert to a different repository by stripping a prefix from the
+ file paths. Also rewrite the message to remove the bug number and reviewer, but add
+ a bugzilla link in the summary.
+
+ :param patch: the Patch to convert
+ :param strip_dir: the path prefix to remove
+ """
+
+ if not strip_dir.startswith("/"):
+ strip_dir = "/%s" % strip_dir
+
+ new_diff = []
+ line_starts = [
+ ("diff ", True),
+ ("+++ ", True),
+ ("--- ", True),
+ ("rename from ", False),
+ ("rename to ", False),
+ ]
+ for line in patch.diff.split("\n"):
+ for start, leading_slash in line_starts:
+ strip = strip_dir if leading_slash else strip_dir[1:]
+ if line.startswith(start):
+ new_diff.append(line.replace(strip, "").encode("utf8"))
+ break
+ else:
+ new_diff.append(line)
+
+ new_diff = "\n".join(new_diff)
+
+ assert new_diff != patch
+
+ return Patch(patch.author, patch.email, rewrite_message(patch), new_diff)
+
+
+def rewrite_message(patch):
+ if patch.message.bug is not None:
+ return "\n".join(
+ [
+ patch.message.summary,
+ patch.message.body,
+ "",
+ "Upstreamed from https://bugzilla.mozilla.org/show_bug.cgi?id=%s [ci skip]"
+ % patch.message.bug, # noqa E501
+ ]
+ )
+
+ return "\n".join(
+ [patch.message.full_summary, "%s\n[ci skip]\n" % patch.message.body]
+ )
+
+
+class SyncToUpstream(Step):
+ """Sync local changes to upstream"""
+
+ def create(self, state):
+ if not state.kwargs["upstream"]:
+ return
+
+ if not isinstance(state.local_tree, GitTree):
+ self.logger.error("Cannot sync with upstream from a non-Git checkout.")
+ return exit_clean
+
+ try:
+ import requests # noqa F401
+ except ImportError:
+ self.logger.error(
+ "Upstream sync requires the requests module to be installed"
+ )
+ return exit_clean
+
+ if not state.sync_tree:
+ os.makedirs(state.sync["path"])
+ state.sync_tree = GitTree(root=state.sync["path"])
+
+ kwargs = state.kwargs
+ with state.push(
+ ["local_tree", "sync_tree", "tests_path", "metadata_path", "sync"]
+ ):
+ state.token = kwargs["token"]
+ runner = SyncToUpstreamRunner(self.logger, state)
+ runner.run()
+
+
+class GetLastSyncData(Step):
+ """Find the gecko commit at which we last performed a sync with upstream and the upstream
+ commit that was synced."""
+
+ provides = ["sync_data_path", "last_sync_commit", "old_upstream_rev"]
+
+ def create(self, state):
+ self.logger.info("Looking for last sync commit")
+ state.sync_data_path = os.path.join(state.metadata_path, "mozilla-sync")
+ items = {}
+ with open(state.sync_data_path) as f:
+ for line in f.readlines():
+ key, value = [item.strip() for item in line.split(":", 1)]
+ items[key] = value
+
+ state.last_sync_commit = Commit(
+ state.local_tree, state.local_tree.rev_from_hg(items["local"])
+ )
+ state.old_upstream_rev = items["upstream"]
+
+ if not state.local_tree.contains_commit(state.last_sync_commit):
+ self.logger.error(
+ "Could not find last sync commit %s" % state.last_sync_commit.sha1
+ )
+ return exit_clean
+
+ self.logger.info(
+ "Last sync to web-platform-tests happened in %s"
+ % state.last_sync_commit.sha1
+ )
+
+
+class CheckoutBranch(Step):
+ """Create a branch in the sync tree pointing at the last upstream sync commit
+ and check it out"""
+
+ provides = ["branch"]
+
+ def create(self, state):
+ self.logger.info("Updating sync tree from %s" % state.sync["remote_url"])
+ state.branch = state.sync_tree.unique_branch_name(
+ "outbound_update_%s" % state.old_upstream_rev
+ )
+ state.sync_tree.update(
+ state.sync["remote_url"], state.sync["branch"], state.branch
+ )
+ state.sync_tree.checkout(state.old_upstream_rev, state.branch, force=True)
+
+
+class GetBaseCommit(Step):
+ """Find the latest upstream commit on the branch that we are syncing with"""
+
+ provides = ["base_commit"]
+
+ def create(self, state):
+ state.base_commit = state.sync_tree.get_remote_sha1(
+ state.sync["remote_url"], state.sync["branch"]
+ )
+ self.logger.debug("New base commit is %s" % state.base_commit.sha1)
+
+
+class LoadCommits(Step):
+ """Get a list of commits in the gecko tree that need to be upstreamed"""
+
+ provides = ["source_commits", "has_backouts"]
+
+ def create(self, state):
+ state.source_commits = state.local_tree.log(
+ state.last_sync_commit, state.tests_path
+ )
+
+ update_regexp = re.compile(
+ "Bug \d+ - Update web-platform-tests to revision [0-9a-f]{40}"
+ )
+
+ state.has_backouts = False
+
+ for i, commit in enumerate(state.source_commits[:]):
+ if update_regexp.match(commit.message.text):
+ # This is a previous update commit so ignore it
+ state.source_commits.remove(commit)
+ continue
+
+ elif commit.message.backouts:
+ # TODO: Add support for collapsing backouts
+ state.has_backouts = True
+
+ elif not commit.message.bug:
+ self.logger.error(
+ "Commit %i (%s) doesn't have an associated bug number."
+ % (i + 1, commit.sha1)
+ )
+ return exit_unclean
+
+ self.logger.debug("Source commits: %s" % state.source_commits)
+
+
+class SelectCommits(Step):
+ """Provide a UI to select which commits to upstream"""
+
+ def create(self, state):
+ while True:
+ commits = state.source_commits[:]
+ for i, commit in enumerate(commits):
+ print("{}:\t{}".format(i, commit.message.summary))
+
+ remove = input(
+ "Provide a space-separated list of any commits numbers "
+ "to remove from the list to upstream:\n"
+ ).strip()
+ remove_idx = set()
+ for item in remove.split(" "):
+ try:
+ item = int(item)
+ except ValueError:
+ continue
+ if item < 0 or item >= len(commits):
+ continue
+ remove_idx.add(item)
+
+ keep_commits = [
+ (i, cmt) for i, cmt in enumerate(commits) if i not in remove_idx
+ ]
+ # TODO: consider printed removed commits
+ print("Selected the following commits to keep:")
+ for i, commit in keep_commits:
+ print("{}:\t{}".format(i, commit.message.summary))
+ confirm = input("Keep the above commits? y/n\n").strip().lower()
+
+ if confirm == "y":
+ state.source_commits = [item[1] for item in keep_commits]
+ break
+
+
+class MovePatches(Step):
+ """Convert gecko commits into patches against upstream and commit these to the sync tree."""
+
+ provides = ["commits_loaded"]
+
+ def create(self, state):
+ if not hasattr(state, "commits_loaded"):
+ state.commits_loaded = 0
+
+ strip_path = os.path.relpath(state.tests_path, state.local_tree.root)
+ self.logger.debug("Stripping patch %s" % strip_path)
+
+ if not hasattr(state, "patch"):
+ state.patch = None
+
+ for commit in state.source_commits[state.commits_loaded :]:
+ i = state.commits_loaded + 1
+ self.logger.info("Moving commit %i: %s" % (i, commit.message.full_summary))
+ stripped_patch = None
+ if state.patch:
+ filename, stripped_patch = state.patch
+ if not os.path.exists(filename):
+ stripped_patch = None
+ else:
+ with open(filename) as f:
+ stripped_patch.diff = f.read()
+ state.patch = None
+ if not stripped_patch:
+ patch = commit.export_patch(state.tests_path)
+ stripped_patch = rewrite_patch(patch, strip_path)
+ if not stripped_patch.diff:
+ self.logger.info("Skipping empty patch")
+ state.commits_loaded = i
+ continue
+ try:
+ state.sync_tree.import_patch(stripped_patch)
+ except Exception:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".diff") as f:
+ f.write(stripped_patch.diff)
+ print(
+ """Patch failed to apply. Diff saved in {}
+Fix this file so it applies and run with --continue""".format(
+ f.name
+ )
+ )
+ state.patch = (f.name, stripped_patch)
+ print(state.patch)
+ sys.exit(1)
+ state.commits_loaded = i
+ input("Check for differences with upstream")
+
+
+class RebaseCommits(Step):
+ """Rebase commits from the current branch on top of the upstream destination branch.
+
+ This step is particularly likely to fail if the rebase generates merge conflicts.
+ In that case the conflicts can be fixed up locally and the sync process restarted
+ with --continue.
+ """
+
+ def create(self, state):
+ self.logger.info("Rebasing local commits")
+ continue_rebase = False
+ # Check if there's a rebase in progress
+ if os.path.exists(
+ os.path.join(state.sync_tree.root, ".git", "rebase-merge")
+ ) or os.path.exists(os.path.join(state.sync_tree.root, ".git", "rebase-apply")):
+ continue_rebase = True
+
+ try:
+ state.sync_tree.rebase(state.base_commit, continue_rebase=continue_rebase)
+ except subprocess.CalledProcessError:
+ self.logger.info(
+ "Rebase failed, fix merge and run %s again with --continue"
+ % sys.argv[0]
+ )
+ raise
+ self.logger.info("Rebase successful")
+
+
+class CheckRebase(Step):
+ """Check if there are any commits remaining after rebase"""
+
+ provides = ["rebased_commits"]
+
+ def create(self, state):
+ state.rebased_commits = state.sync_tree.log(state.base_commit)
+ if not state.rebased_commits:
+ self.logger.info("Nothing to upstream, exiting")
+ return exit_clean
+
+
+class MergeUpstream(Step):
+ """Run steps to push local commits as seperate PRs and merge upstream."""
+
+ provides = ["merge_index", "gh_repo"]
+
+ def create(self, state):
+ gh = GitHub(state.token)
+ if "merge_index" not in state:
+ state.merge_index = 0
+
+ org, name = urlparse.urlsplit(state.sync["remote_url"]).path[1:].split("/")
+ if name.endswith(".git"):
+ name = name[:-4]
+ state.gh_repo = gh.repo(org, name)
+ for commit in state.rebased_commits[state.merge_index :]:
+ with state.push(["gh_repo", "sync_tree"]):
+ state.commit = commit
+ pr_merger = PRMergeRunner(self.logger, state)
+ rv = pr_merger.run()
+ if rv is not None:
+ return rv
+ state.merge_index += 1
+
+
+class UpdateLastSyncData(Step):
+ """Update the gecko commit at which we last performed a sync with upstream."""
+
+ provides = []
+
+ def create(self, state):
+ self.logger.info("Updating last sync commit")
+ data = {
+ "local": state.local_tree.rev_to_hg(state.local_tree.rev),
+ "upstream": state.sync_tree.rev,
+ }
+ with open(state.sync_data_path, "w") as f:
+ for key, value in data.iteritems():
+ f.write("%s: %s\n" % (key, value))
+ # This gets added to the patch later on
+
+
+class MergeLocalBranch(Step):
+ """Create a local branch pointing at the commit to upstream"""
+
+ provides = ["local_branch"]
+
+ def create(self, state):
+ branch_prefix = "sync_%s" % state.commit.sha1
+ local_branch = state.sync_tree.unique_branch_name(branch_prefix)
+
+ state.sync_tree.create_branch(local_branch, state.commit)
+ state.local_branch = local_branch
+
+
+class MergeRemoteBranch(Step):
+ """Get an unused remote branch name to use for the PR"""
+
+ provides = ["remote_branch"]
+
+ def create(self, state):
+ remote_branch = "sync_%s" % state.commit.sha1
+ branches = [
+ ref[len("refs/heads/") :]
+ for sha1, ref in state.sync_tree.list_remote(state.gh_repo.url)
+ if ref.startswith("refs/heads")
+ ]
+ state.remote_branch = get_unique_name(branches, remote_branch)
+
+
+class PushUpstream(Step):
+ """Push local branch to remote"""
+
+ def create(self, state):
+ self.logger.info("Pushing commit upstream")
+ state.sync_tree.push(state.gh_repo.url, state.local_branch, state.remote_branch)
+
+
+class CreatePR(Step):
+ """Create a PR for the remote branch"""
+
+ provides = ["pr"]
+
+ def create(self, state):
+ self.logger.info("Creating a PR")
+ commit = state.commit
+ state.pr = state.gh_repo.create_pr(
+ commit.message.full_summary,
+ state.remote_branch,
+ "master",
+ commit.message.body if commit.message.body else "",
+ )
+
+
+class PRAddComment(Step):
+ """Add an issue comment indicating that the code has been reviewed already"""
+
+ def create(self, state):
+ state.pr.issue.add_comment("Code reviewed upstream.")
+
+
+class MergePR(Step):
+ """Merge the PR"""
+
+ def create(self, state):
+ self.logger.info("Merging PR")
+ state.pr.merge()
+
+
+class PRDeleteBranch(Step):
+ """Delete the remote branch"""
+
+ def create(self, state):
+ self.logger.info("Deleting remote branch")
+ state.sync_tree.push(state.gh_repo.url, "", state.remote_branch)
+
+
+class SyncToUpstreamRunner(StepRunner):
+ """Runner for syncing local changes to upstream"""
+
+ steps = [
+ GetLastSyncData,
+ CheckoutBranch,
+ GetBaseCommit,
+ LoadCommits,
+ SelectCommits,
+ MovePatches,
+ RebaseCommits,
+ CheckRebase,
+ MergeUpstream,
+ UpdateLastSyncData,
+ ]
+
+
+class PRMergeRunner(StepRunner):
+ """(Sub)Runner for creating and merging a PR"""
+
+ steps = [
+ MergeLocalBranch,
+ MergeRemoteBranch,
+ PushUpstream,
+ CreatePR,
+ PRAddComment,
+ MergePR,
+ PRDeleteBranch,
+ ]
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-02 12:06:01 +0000