summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/update/upstream.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/web-platform/update/upstream.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/web-platform/update/upstream.py')
-rw-r--r--testing/web-platform/update/upstream.py407
1 files changed, 407 insertions, 0 deletions
diff --git a/testing/web-platform/update/upstream.py b/testing/web-platform/update/upstream.py
new file mode 100644
index 000000000..70ae09cd8
--- /dev/null
+++ b/testing/web-platform/update/upstream.py
@@ -0,0 +1,407 @@
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import urlparse
+
+from wptrunner.update.sync import LoadManifest
+from wptrunner.update.tree import get_unique_name
+from wptrunner.update.base import Step, StepRunner, exit_clean, exit_unclean
+
+from .tree import Commit, GitTree, Patch
+import github
+from .github import GitHub
+
+
+def rewrite_patch(patch, strip_dir):
+ """Take a Patch and convert to a different repository by stripping a prefix from the
+ file paths. Also rewrite the message to remove the bug number and reviewer, but add
+ a bugzilla link in the summary.
+
+ :param patch: the Patch to convert
+ :param strip_dir: the path prefix to remove
+ """
+
+ if not strip_dir.startswith("/"):
+ strip_dir = "/%s"% strip_dir
+
+ new_diff = []
+ line_starts = ["diff ", "+++ ", "--- ", "rename from ", "rename to "]
+ for line in patch.diff.split("\n"):
+ for start in line_starts:
+ if line.startswith(start):
+ new_diff.append(line.replace(strip_dir, "").encode("utf8"))
+ break
+ else:
+ new_diff.append(line)
+
+ new_diff = "\n".join(new_diff)
+
+ assert new_diff != patch
+
+ return Patch(patch.author, patch.email, rewrite_message(patch), new_diff)
+
+def rewrite_message(patch):
+ rest = patch.message.body
+
+ if patch.message.bug is not None:
+ return "\n".join([patch.message.summary,
+ patch.message.body,
+ "",
+ "Upstreamed from https://bugzilla.mozilla.org/show_bug.cgi?id=%s" %
+ patch.message.bug])
+
+ return "\n".join([patch.message.full_summary, rest])
+
+
+class SyncToUpstream(Step):
+ """Sync local changes to upstream"""
+
+ def create(self, state):
+ if not state.kwargs["upstream"]:
+ return
+
+ if not isinstance(state.local_tree, GitTree):
+ self.logger.error("Cannot sync with upstream from a non-Git checkout.")
+ return exit_clean
+
+ try:
+ import requests
+ except ImportError:
+ self.logger.error("Upstream sync requires the requests module to be installed")
+ return exit_clean
+
+ if not state.sync_tree:
+ os.makedirs(state.sync["path"])
+ state.sync_tree = GitTree(root=state.sync["path"])
+
+ kwargs = state.kwargs
+ with state.push(["local_tree", "sync_tree", "tests_path", "metadata_path",
+ "sync"]):
+ state.token = kwargs["token"]
+ runner = SyncToUpstreamRunner(self.logger, state)
+ runner.run()
+
+
+class CheckoutBranch(Step):
+ """Create a branch in the sync tree pointing at the last upstream sync commit
+ and check it out"""
+
+ provides = ["branch"]
+
+ def create(self, state):
+ self.logger.info("Updating sync tree from %s" % state.sync["remote_url"])
+ state.branch = state.sync_tree.unique_branch_name(
+ "outbound_update_%s" % state.old_manifest.rev)
+ state.sync_tree.update(state.sync["remote_url"],
+ state.sync["branch"],
+ state.branch)
+ state.sync_tree.checkout(state.old_manifest.rev, state.branch, force=True)
+
+
+class GetLastSyncCommit(Step):
+ """Find the gecko commit at which we last performed a sync with upstream."""
+
+ provides = ["last_sync_path", "last_sync_commit"]
+
+ def create(self, state):
+ self.logger.info("Looking for last sync commit")
+ state.last_sync_path = os.path.join(state.metadata_path, "mozilla-sync")
+ with open(state.last_sync_path) as f:
+ last_sync_sha1 = f.read().strip()
+
+ state.last_sync_commit = Commit(state.local_tree, last_sync_sha1)
+
+ if not state.local_tree.contains_commit(state.last_sync_commit):
+ self.logger.error("Could not find last sync commit %s" % last_sync_sha1)
+ return exit_clean
+
+ self.logger.info("Last sync to web-platform-tests happened in %s" % state.last_sync_commit.sha1)
+
+
+class GetBaseCommit(Step):
+ """Find the latest upstream commit on the branch that we are syncing with"""
+
+ provides = ["base_commit"]
+
+ def create(self, state):
+ state.base_commit = state.sync_tree.get_remote_sha1(state.sync["remote_url"],
+ state.sync["branch"])
+ self.logger.debug("New base commit is %s" % state.base_commit.sha1)
+
+
+class LoadCommits(Step):
+ """Get a list of commits in the gecko tree that need to be upstreamed"""
+
+ provides = ["source_commits", "has_backouts"]
+
+ def create(self, state):
+ state.source_commits = state.local_tree.log(state.last_sync_commit,
+ state.tests_path)
+
+ update_regexp = re.compile("Bug \d+ - Update web-platform-tests to revision [0-9a-f]{40}")
+
+ state.has_backouts = False
+
+ for i, commit in enumerate(state.source_commits[:]):
+ if update_regexp.match(commit.message.text):
+ # This is a previous update commit so ignore it
+ state.source_commits.remove(commit)
+ continue
+
+ elif commit.message.backouts:
+ #TODO: Add support for collapsing backouts
+ state.has_backouts = True
+
+ elif not commit.message.bug:
+ self.logger.error("Commit %i (%s) doesn't have an associated bug number." %
+ (i + 1, commit.sha1))
+ return exit_unclean
+
+ self.logger.debug("Source commits: %s" % state.source_commits)
+
+class SelectCommits(Step):
+ """Provide a UI to select which commits to upstream"""
+
+ def create(self, state):
+ while True:
+ commits = state.source_commits[:]
+ for i, commit in enumerate(commits):
+ print "%i:\t%s" % (i, commit.message.summary)
+
+ remove = raw_input("Provide a space-separated list of any commits numbers to remove from the list to upstream:\n").strip()
+ remove_idx = set()
+ for item in remove.split(" "):
+ try:
+ item = int(item)
+ except:
+ continue
+ if item < 0 or item >= len(commits):
+ continue
+ remove_idx.add(item)
+
+ keep_commits = [(i,cmt) for i,cmt in enumerate(commits) if i not in remove_idx]
+ #TODO: consider printed removed commits
+ print "Selected the following commits to keep:"
+ for i, commit in keep_commits:
+ print "%i:\t%s" % (i, commit.message.summary)
+ confirm = raw_input("Keep the above commits? y/n\n").strip().lower()
+
+ if confirm == "y":
+ state.source_commits = [item[1] for item in keep_commits]
+ break
+
+class MovePatches(Step):
+ """Convert gecko commits into patches against upstream and commit these to the sync tree."""
+
+ provides = ["commits_loaded"]
+
+ def create(self, state):
+ if not hasattr(state, "commits_loaded"):
+ state.commits_loaded = 0
+
+ strip_path = os.path.relpath(state.tests_path,
+ state.local_tree.root)
+ self.logger.debug("Stripping patch %s" % strip_path)
+
+ if not hasattr(state, "patch"):
+ state.patch = None
+
+ for commit in state.source_commits[state.commits_loaded:]:
+ i = state.commits_loaded + 1
+ self.logger.info("Moving commit %i: %s" % (i, commit.message.full_summary))
+ stripped_patch = None
+ if state.patch:
+ filename, stripped_patch = state.patch
+ if not os.path.exists(filename):
+ stripped_patch = None
+ else:
+ with open(filename) as f:
+ stripped_patch.diff = f.read()
+ state.patch = None
+ if not stripped_patch:
+ patch = commit.export_patch(state.tests_path)
+ stripped_patch = rewrite_patch(patch, strip_path)
+ if not stripped_patch.diff:
+ self.logger.info("Skipping empty patch")
+ state.commits_loaded = i
+ continue
+ try:
+ state.sync_tree.import_patch(stripped_patch)
+ except:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".diff") as f:
+ f.write(stripped_patch.diff)
+ print """Patch failed to apply. Diff saved in %s
+Fix this file so it applies and run with --continue""" % f.name
+ state.patch = (f.name, stripped_patch)
+ print state.patch
+ sys.exit(1)
+ state.commits_loaded = i
+ raw_input("Check for differences with upstream")
+
+
+class RebaseCommits(Step):
+ """Rebase commits from the current branch on top of the upstream destination branch.
+
+ This step is particularly likely to fail if the rebase generates merge conflicts.
+ In that case the conflicts can be fixed up locally and the sync process restarted
+ with --continue.
+ """
+ def create(self, state):
+ self.logger.info("Rebasing local commits")
+ continue_rebase = False
+ # Check if there's a rebase in progress
+ if (os.path.exists(os.path.join(state.sync_tree.root,
+ ".git",
+ "rebase-merge")) or
+ os.path.exists(os.path.join(state.sync_tree.root,
+ ".git",
+ "rebase-apply"))):
+ continue_rebase = True
+
+ try:
+ state.sync_tree.rebase(state.base_commit, continue_rebase=continue_rebase)
+ except subprocess.CalledProcessError:
+ self.logger.info("Rebase failed, fix merge and run %s again with --continue" % sys.argv[0])
+ raise
+ self.logger.info("Rebase successful")
+
+class CheckRebase(Step):
+ """Check if there are any commits remaining after rebase"""
+ provides = ["rebased_commits"]
+
+ def create(self, state):
+ state.rebased_commits = state.sync_tree.log(state.base_commit)
+ if not state.rebased_commits:
+ self.logger.info("Nothing to upstream, exiting")
+ return exit_clean
+
+class MergeUpstream(Step):
+ """Run steps to push local commits as seperate PRs and merge upstream."""
+
+ provides = ["merge_index", "gh_repo"]
+
+ def create(self, state):
+ gh = GitHub(state.token)
+ if "merge_index" not in state:
+ state.merge_index = 0
+
+ org, name = urlparse.urlsplit(state.sync["remote_url"]).path[1:].split("/")
+ if name.endswith(".git"):
+ name = name[:-4]
+ state.gh_repo = gh.repo(org, name)
+ for commit in state.rebased_commits[state.merge_index:]:
+ with state.push(["gh_repo", "sync_tree"]):
+ state.commit = commit
+ pr_merger = PRMergeRunner(self.logger, state)
+ rv = pr_merger.run()
+ if rv is not None:
+ return rv
+ state.merge_index += 1
+
+class UpdateLastSyncCommit(Step):
+ """Update the gecko commit at which we last performed a sync with upstream."""
+
+ provides = []
+
+ def create(self, state):
+ self.logger.info("Updating last sync commit")
+ with open(state.last_sync_path, "w") as f:
+ f.write(state.local_tree.rev)
+ # This gets added to the patch later on
+
+class MergeLocalBranch(Step):
+ """Create a local branch pointing at the commit to upstream"""
+
+ provides = ["local_branch"]
+
+ def create(self, state):
+ branch_prefix = "sync_%s" % state.commit.sha1
+ local_branch = state.sync_tree.unique_branch_name(branch_prefix)
+
+ state.sync_tree.create_branch(local_branch, state.commit)
+ state.local_branch = local_branch
+
+class MergeRemoteBranch(Step):
+ """Get an unused remote branch name to use for the PR"""
+ provides = ["remote_branch"]
+
+ def create(self, state):
+ remote_branch = "sync_%s" % state.commit.sha1
+ branches = [ref[len("refs/heads/"):] for sha1, ref in
+ state.sync_tree.list_remote(state.gh_repo.url)
+ if ref.startswith("refs/heads")]
+ state.remote_branch = get_unique_name(branches, remote_branch)
+
+
+class PushUpstream(Step):
+ """Push local branch to remote"""
+ def create(self, state):
+ self.logger.info("Pushing commit upstream")
+ state.sync_tree.push(state.gh_repo.url,
+ state.local_branch,
+ state.remote_branch)
+
+class CreatePR(Step):
+ """Create a PR for the remote branch"""
+
+ provides = ["pr"]
+
+ def create(self, state):
+ self.logger.info("Creating a PR")
+ commit = state.commit
+ state.pr = state.gh_repo.create_pr(commit.message.full_summary,
+ state.remote_branch,
+ "master",
+ commit.message.body if commit.message.body else "")
+
+
+class PRAddComment(Step):
+ """Add an issue comment indicating that the code has been reviewed already"""
+ def create(self, state):
+ state.pr.issue.add_comment("Code reviewed upstream.")
+
+
+class MergePR(Step):
+ """Merge the PR"""
+
+ def create(self, state):
+ self.logger.info("Merging PR")
+ state.pr.merge()
+
+
+class PRDeleteBranch(Step):
+ """Delete the remote branch"""
+
+ def create(self, state):
+ self.logger.info("Deleting remote branch")
+ state.sync_tree.push(state.gh_repo.url, "", state.remote_branch)
+
+
+class SyncToUpstreamRunner(StepRunner):
+ """Runner for syncing local changes to upstream"""
+ steps = [LoadManifest,
+ CheckoutBranch,
+ GetLastSyncCommit,
+ GetBaseCommit,
+ LoadCommits,
+ SelectCommits,
+ MovePatches,
+ RebaseCommits,
+ CheckRebase,
+ MergeUpstream,
+ UpdateLastSyncCommit]
+
+
+class PRMergeRunner(StepRunner):
+ """(Sub)Runner for creating and merging a PR"""
+ steps = [
+ MergeLocalBranch,
+ MergeRemoteBranch,
+ PushUpstream,
+ CreatePR,
+ PRAddComment,
+ MergePR,
+ PRDeleteBranch,
+ ]