summaryrefslogtreecommitdiffstats
path: root/testing/mozharness/scripts/merge_day/gecko_migration.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/mozharness/scripts/merge_day/gecko_migration.py')
-rwxr-xr-xtesting/mozharness/scripts/merge_day/gecko_migration.py545
1 files changed, 545 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/merge_day/gecko_migration.py b/testing/mozharness/scripts/merge_day/gecko_migration.py
new file mode 100755
index 000000000..7208630e0
--- /dev/null
+++ b/testing/mozharness/scripts/merge_day/gecko_migration.py
@@ -0,0 +1,545 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" gecko_migration.py
+
+Merge day script for gecko (mozilla-central -> mozilla-aurora,
+mozilla-aurora -> mozilla-beta, mozilla-beta -> mozilla-release).
+
+Ported largely from
+http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/beta2release.py
+and
+http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/merge_helper.py
+"""
+
+import os
+import pprint
+import subprocess
+import sys
+from getpass import getpass
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.errors import HgErrorList
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.selfserve import SelfServeMixin
+from mozharness.mozilla.updates.balrog import BalrogMixin
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
+
+VALID_MIGRATION_BEHAVIORS = (
+ "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr",
+ "bump_second_digit",
+)
+
+
+# GeckoMigration {{{1
+class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin,
+ SelfServeMixin, BuildbotMixin,
+ MercurialRepoManipulationMixin):
+ config_options = [
+ [['--hg-user', ], {
+ "action": "store",
+ "dest": "hg_user",
+ "type": "string",
+ "default": "ffxbld <release@mozilla.com>",
+ "help": "Specify what user to use to commit to hg.",
+ }],
+ [['--balrog-api-root', ], {
+ "action": "store",
+ "dest": "balrog_api_root",
+ "type": "string",
+ "help": "Specify Balrog API root URL.",
+ }],
+ [['--balrog-username', ], {
+ "action": "store",
+ "dest": "balrog_username",
+ "type": "string",
+ "help": "Specify what user to connect to Balrog with.",
+ }],
+ [['--balrog-credentials-file', ], {
+ "action": "store",
+ "dest": "balrog_credentials_file",
+ "type": "string",
+ "help": "The file containing the Balrog credentials.",
+ }],
+ [['--remove-locale', ], {
+ "action": "extend",
+ "dest": "remove_locales",
+ "type": "string",
+ "help": "Comma separated list of locales to remove from the 'to' repo.",
+ }],
+ ]
+ gecko_repos = None
+
+ def __init__(self, require_config_file=True):
+ super(GeckoMigration, self).__init__(
+ config_options=virtualenv_config_options + self.config_options,
+ all_actions=[
+ 'clobber',
+ 'create-virtualenv',
+ 'clean-repos',
+ 'pull',
+ 'lock-update-paths',
+ 'migrate',
+ 'bump_second_digit',
+ 'commit-changes',
+ 'push',
+ 'trigger-builders',
+ ],
+ default_actions=[
+ 'clean-repos',
+ 'pull',
+ 'migrate',
+ ],
+ require_config_file=require_config_file
+ )
+ self.run_sanity_check()
+
+# Helper methods {{{1
+ def run_sanity_check(self):
+ """ Verify the configs look sane before proceeding.
+ """
+ message = ""
+ if self.config['migration_behavior'] not in VALID_MIGRATION_BEHAVIORS:
+ message += "%s must be one of %s!\n" % (self.config['migration_behavior'], VALID_MIGRATION_BEHAVIORS)
+ if self.config['migration_behavior'] == 'beta_to_release':
+ if self.config.get("require_remove_locales") and not self.config.get("remove_locales") and 'migrate' in self.actions:
+ message += "You must specify --remove-locale!\n"
+ else:
+ if self.config.get("require_remove_locales") or self.config.get("remove_locales"):
+ self.warning("--remove-locale isn't valid unless you're using beta_to_release migration_behavior!\n")
+ if message:
+ self.fatal(message)
+
+ def query_abs_dirs(self):
+ """ Allow for abs_from_dir and abs_to_dir
+ """
+ if self.abs_dirs:
+ return self.abs_dirs
+ dirs = super(GeckoMigration, self).query_abs_dirs()
+ self.abs_dirs['abs_tools_dir'] = os.path.join(
+ dirs['abs_work_dir'], 'tools'
+ )
+ self.abs_dirs['abs_tools_lib_dir'] = os.path.join(
+ dirs['abs_work_dir'], 'tools', 'lib', 'python'
+ )
+ for k in ('from', 'to'):
+ url = self.config.get("%s_repo_url" % k)
+ if url:
+ dir_name = self.get_filename_from_url(url)
+ self.info("adding %s" % dir_name)
+ self.abs_dirs['abs_%s_dir' % k] = os.path.join(
+ dirs['abs_work_dir'], dir_name
+ )
+ return self.abs_dirs
+
+ def query_repos(self):
+ """ Build a list of repos to clone.
+ """
+ if self.gecko_repos:
+ return self.gecko_repos
+ self.info("Building gecko_repos list...")
+ dirs = self.query_abs_dirs()
+ self.gecko_repos = []
+ for k in ('from', 'to'):
+ repo_key = "%s_repo_url" % k
+ url = self.config.get(repo_key)
+ if url:
+ self.gecko_repos.append({
+ "repo": url,
+ "branch": self.config.get("%s_repo_branch" % (k,), "default"),
+ "dest": dirs['abs_%s_dir' % k],
+ "vcs": "hg",
+ # "hg" vcs uses robustcheckout extension requires the use of a share
+ # but having a share breaks migration logic when merging repos.
+ # Solution: tell hg vcs to create a unique share directory for each
+ # gecko repo. see mozharness/base/vcs/mercurial.py for implementation
+ "use_vcs_unique_share": True,
+ })
+ else:
+ self.warning("Skipping %s" % repo_key)
+ self.info(pprint.pformat(self.gecko_repos))
+ return self.gecko_repos
+
+ def query_commit_dirs(self):
+ dirs = self.query_abs_dirs()
+ commit_dirs = [dirs['abs_to_dir']]
+ if self.config['migration_behavior'] == 'central_to_aurora':
+ commit_dirs.append(dirs['abs_from_dir'])
+ return commit_dirs
+
+ def query_commit_message(self):
+ return "Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release"
+
+ def query_push_dirs(self):
+ dirs = self.query_abs_dirs()
+ return dirs.get('abs_from_dir'), dirs.get('abs_to_dir')
+
+ def query_push_args(self, cwd):
+ if cwd == self.query_abs_dirs()['abs_to_dir'] and \
+ self.config['migration_behavior'] == 'beta_to_release':
+ return ['--new-branch', '-r', '.']
+ else:
+ return ['-r', '.']
+
+ def query_from_revision(self):
+ """ Shortcut to get the revision for the from repo
+ """
+ dirs = self.query_abs_dirs()
+ return self.query_hg_revision(dirs['abs_from_dir'])
+
+ def query_to_revision(self):
+ """ Shortcut to get the revision for the to repo
+ """
+ dirs = self.query_abs_dirs()
+ return self.query_hg_revision(dirs['abs_to_dir'])
+
+ def hg_merge_via_debugsetparents(self, cwd, old_head, new_head,
+ preserve_tags=True, user=None):
+ """ Merge 2 heads avoiding non-fastforward commits
+ """
+ hg = self.query_exe('hg', return_type='list')
+ cmd = hg + ['debugsetparents', new_head, old_head]
+ self.run_command(cmd, cwd=cwd, error_list=HgErrorList,
+ halt_on_failure=True)
+ self.hg_commit(
+ cwd,
+ message="Merge old head via |hg debugsetparents %s %s|. "
+ "CLOSED TREE DONTBUILD a=release" % (new_head, old_head),
+ user=user
+ )
+ if preserve_tags:
+ # I don't know how to do this elegantly.
+ # I'm reverting .hgtags to old_head, then appending the new tags
+ # from new_head to .hgtags, and hoping nothing goes wrong.
+ # I'd rather not write patch files from scratch, so this seems
+ # like a slightly more complex but less objectionable method?
+ self.info("Trying to preserve tags from before debugsetparents...")
+ dirs = self.query_abs_dirs()
+ patch_file = os.path.join(dirs['abs_work_dir'], 'patch_file')
+ self.run_command(
+ subprocess.list2cmdline(hg + ['diff', '-r', old_head, '.hgtags', '-U9', '>', patch_file]),
+ cwd=cwd,
+ )
+ self.run_command(
+ ['patch', '-R', '-p1', '-i', patch_file],
+ cwd=cwd,
+ halt_on_failure=True,
+ )
+ tag_diff = self.read_from_file(patch_file)
+ with self.opened(os.path.join(cwd, '.hgtags'), open_mode='a') as (fh, err):
+ if err:
+ self.fatal("Can't append to .hgtags!")
+ for n, line in enumerate(tag_diff.splitlines()):
+ # The first 4 lines of a patch are headers, so we ignore them.
+ if n < 5:
+ continue
+ # Even after that, the only lines we really care about are
+ # additions to the file.
+ # TODO: why do we only care about additions? I couldn't
+ # figure that out by reading this code.
+ if not line.startswith('+'):
+ continue
+ line = line.replace('+', '')
+ (changeset, tag) = line.split(' ')
+ if len(changeset) != 40:
+ continue
+ fh.write("%s\n" % line)
+ out = self.get_output_from_command(['hg', 'status', '.hgtags'],
+ cwd=cwd)
+ if out:
+ self.hg_commit(
+ cwd,
+ message="Preserve old tags after debugsetparents. "
+ "CLOSED TREE DONTBUILD a=release",
+ user=user,
+ )
+ else:
+ self.info(".hgtags file is identical, no need to commit")
+
+ def remove_locales(self, file_name, locales):
+ """ Remove locales from shipped-locales (m-r only)
+ """
+ contents = self.read_from_file(file_name)
+ new_contents = ""
+ for line in contents.splitlines():
+ locale = line.split()[0]
+ if locale not in locales:
+ new_contents += "%s\n" % line
+ else:
+ self.info("Removed locale: %s" % locale)
+ self.write_to_file(file_name, new_contents)
+
+ def touch_clobber_file(self, cwd):
+ clobber_file = os.path.join(cwd, 'CLOBBER')
+ contents = self.read_from_file(clobber_file)
+ new_contents = ""
+ for line in contents.splitlines():
+ line = line.strip()
+ if line.startswith("#") or line == '':
+ new_contents += "%s\n" % line
+ new_contents += "Merge day clobber"
+ self.write_to_file(clobber_file, new_contents)
+
+ def bump_version(self, cwd, curr_version, next_version, curr_suffix,
+ next_suffix, bump_major=False):
+ """ Bump versions (m-c, m-a, m-b).
+
+ At some point we may want to unhardcode these filenames into config
+ """
+ curr_weave_version = str(int(curr_version) + 2)
+ next_weave_version = str(int(curr_weave_version) + 1)
+ for f in self.config["version_files"]:
+ from_ = "%s.0%s" % (curr_version, curr_suffix)
+ to = "%s.0%s%s" % (next_version, next_suffix, f["suffix"])
+ self.replace(os.path.join(cwd, f["file"]), from_, to)
+
+ # only applicable for m-c
+ if bump_major:
+ self.replace(
+ os.path.join(cwd, "xpcom/components/Module.h"),
+ "static const unsigned int kVersion = %s;" % curr_version,
+ "static const unsigned int kVersion = %s;" % next_version
+ )
+ self.replace(
+ os.path.join(cwd, "services/sync/moz.build"),
+ "DEFINES['weave_version'] = '1.%s.0'" % curr_weave_version,
+ "DEFINES['weave_version'] = '1.%s.0'" % next_weave_version
+ )
+
+ # Branch-specific workflow helper methods {{{1
+ def central_to_aurora(self, end_tag):
+ """ mozilla-central -> mozilla-aurora behavior.
+
+ We could have all of these individually toggled by flags, but
+ by separating into workflow methods we can be more precise about
+ what happens in each workflow, while allowing for things like
+ staging beta user repo migrations.
+ """
+ dirs = self.query_abs_dirs()
+ self.info("Reverting locales")
+ hg = self.query_exe("hg", return_type="list")
+ for f in self.config["locale_files"]:
+ self.run_command(
+ hg + ["revert", "-r", end_tag, f],
+ cwd=dirs['abs_to_dir'],
+ error_list=HgErrorList,
+ halt_on_failure=True,
+ )
+ next_ma_version = self.get_version(dirs['abs_to_dir'])[0]
+ self.bump_version(dirs['abs_to_dir'], next_ma_version, next_ma_version, "a1", "a2")
+ self.apply_replacements()
+ # bump m-c version
+ curr_mc_version = self.get_version(dirs['abs_from_dir'])[0]
+ next_mc_version = str(int(curr_mc_version) + 1)
+ self.bump_version(
+ dirs['abs_from_dir'], curr_mc_version, next_mc_version, "a1", "a1",
+ bump_major=True
+ )
+ # touch clobber files
+ self.touch_clobber_file(dirs['abs_from_dir'])
+ self.touch_clobber_file(dirs['abs_to_dir'])
+
+ def aurora_to_beta(self, *args, **kwargs):
+ """ mozilla-aurora -> mozilla-beta behavior.
+
+ We could have all of these individually toggled by flags, but
+ by separating into workflow methods we can be more precise about
+ what happens in each workflow, while allowing for things like
+ staging beta user repo migrations.
+ """
+ dirs = self.query_abs_dirs()
+ mb_version = self.get_version(dirs['abs_to_dir'])[0]
+ self.bump_version(dirs['abs_to_dir'], mb_version, mb_version, "a2", "")
+ self.apply_replacements()
+ self.touch_clobber_file(dirs['abs_to_dir'])
+ # TODO mozconfig diffing
+ # The build/tools version only checks the mozconfigs from hgweb, so
+ # can't help pre-push. The in-tree mozconfig diffing requires a mach
+ # virtualenv to be installed. If we want this sooner we can put this
+ # in the push action; otherwise we may just wait until we have in-tree
+ # mozconfig checking.
+
+ def beta_to_release(self, *args, **kwargs):
+ """ mozilla-beta -> mozilla-release behavior.
+
+ We could have all of these individually toggled by flags, but
+ by separating into workflow methods we can be more precise about
+ what happens in each workflow, while allowing for things like
+ staging beta user repo migrations.
+ """
+ dirs = self.query_abs_dirs()
+ # Reset display_version.txt
+ for f in self.config["copy_files"]:
+ self.copyfile(
+ os.path.join(dirs['abs_to_dir'], f["src"]),
+ os.path.join(dirs['abs_to_dir'], f["dst"]))
+
+ self.apply_replacements()
+ if self.config.get("remove_locales"):
+ self.remove_locales(
+ os.path.join(dirs['abs_to_dir'], "browser/locales/shipped-locales"),
+ self.config['remove_locales']
+ )
+ self.touch_clobber_file(dirs['abs_to_dir'])
+
+ def release_to_esr(self, *args, **kwargs):
+ """ mozilla-release -> mozilla-esrNN behavior. """
+ dirs = self.query_abs_dirs()
+ for to_transplant in self.config.get("transplant_patches", []):
+ self.transplant(repo=to_transplant["repo"],
+ changeset=to_transplant["changeset"],
+ cwd=dirs['abs_to_dir'])
+ self.apply_replacements()
+ self.touch_clobber_file(dirs['abs_to_dir'])
+
+ def apply_replacements(self):
+ dirs = self.query_abs_dirs()
+ for f, from_, to in self.config["replacements"]:
+ self.replace(os.path.join(dirs['abs_to_dir'], f), from_, to)
+
+ def transplant(self, repo, changeset, cwd):
+ """Transplant a Mercurial changeset from a remote repository."""
+ hg = self.query_exe("hg", return_type="list")
+ cmd = hg + ["--config", "extensions.transplant=", "transplant",
+ "--source", repo, changeset]
+ self.info("Transplanting %s from %s" % (changeset, repo))
+ status = self.run_command(
+ cmd,
+ cwd=cwd,
+ error_list=HgErrorList,
+ )
+ if status != 0:
+ self.fatal("Cannot transplant %s from %s properly" %
+ (changeset, repo))
+
+ def pull_from_repo(self, from_dir, to_dir, revision=None, branch=None):
+ """ Pull from one repo to another. """
+ hg = self.query_exe("hg", return_type="list")
+ cmd = hg + ["pull"]
+ if revision:
+ cmd.extend(["-r", revision])
+ cmd.append(from_dir)
+ self.run_command(
+ cmd,
+ cwd=to_dir,
+ error_list=HgErrorList,
+ halt_on_failure=True,
+ )
+ cmd = hg + ["update", "-C"]
+ if branch or revision:
+ cmd.extend(["-r", branch or revision])
+ self.run_command(
+ cmd,
+ cwd=to_dir,
+ error_list=HgErrorList,
+ halt_on_failure=True,
+ )
+
+# Actions {{{1
+ def bump_second_digit(self, *args, **kwargs):
+ """Bump second digit.
+
+ ESR need only the second digit bumped as a part of merge day."""
+ dirs = self.query_abs_dirs()
+ version = self.get_version(dirs['abs_to_dir'])
+ curr_version = ".".join(version)
+ next_version = list(version)
+ # bump the second digit
+ next_version[1] = str(int(next_version[1]) + 1)
+ # Take major+minor and append '0' accordng to Firefox version schema.
+ # 52.0 will become 52.1.0, not 52.1
+ next_version = ".".join(next_version[:2] + ['0'])
+ for f in self.config["version_files"]:
+ self.replace(os.path.join(dirs['abs_to_dir'], f["file"]),
+ curr_version, next_version + f["suffix"])
+ self.touch_clobber_file(dirs['abs_to_dir'])
+
+ def pull(self):
+ """ Pull tools first, then clone the gecko repos
+ """
+ repos = [{
+ "repo": self.config["tools_repo_url"],
+ "branch": self.config["tools_repo_branch"],
+ "dest": "tools",
+ "vcs": "hg",
+ }] + self.query_repos()
+ super(GeckoMigration, self).pull(repos=repos)
+
+ def lock_update_paths(self):
+ self.lock_balrog_rules(self.config["balrog_rules_to_lock"])
+
+ def migrate(self):
+ """ Perform the migration.
+ """
+ dirs = self.query_abs_dirs()
+ from_fx_major_version = self.get_version(dirs['abs_from_dir'])[0]
+ to_fx_major_version = self.get_version(dirs['abs_to_dir'])[0]
+ base_from_rev = self.query_from_revision()
+ base_to_rev = self.query_to_revision()
+ base_tag = self.config['base_tag'] % {'major_version': from_fx_major_version}
+ end_tag = self.config['end_tag'] % {'major_version': to_fx_major_version}
+ self.hg_tag(
+ dirs['abs_from_dir'], base_tag, user=self.config['hg_user'],
+ revision=base_from_rev,
+ )
+ new_from_rev = self.query_from_revision()
+ self.info("New revision %s" % new_from_rev)
+ pull_revision = None
+ if not self.config.get("pull_all_branches"):
+ pull_revision = new_from_rev
+ self.pull_from_repo(
+ dirs['abs_from_dir'], dirs['abs_to_dir'],
+ revision=pull_revision,
+ branch="default",
+ )
+ if self.config.get("requires_head_merge") is not False:
+ self.hg_merge_via_debugsetparents(
+ dirs['abs_to_dir'], old_head=base_to_rev, new_head=new_from_rev,
+ user=self.config['hg_user'],
+ )
+ self.hg_tag(
+ dirs['abs_to_dir'], end_tag, user=self.config['hg_user'],
+ revision=base_to_rev, force=True,
+ )
+ # Call beta_to_release etc.
+ if not hasattr(self, self.config['migration_behavior']):
+ self.fatal("Don't know how to proceed with migration_behavior %s !" % self.config['migration_behavior'])
+ getattr(self, self.config['migration_behavior'])(end_tag=end_tag)
+ self.info("Verify the diff, and apply any manual changes, such as disabling features, and --commit-changes")
+
+ def trigger_builders(self):
+ """Triggers builders that should be run directly after a merge.
+ There are two different types of things we trigger:
+ 1) Nightly builds ("post_merge_nightly_branches" in the config).
+ These are triggered with buildapi's nightly build endpoint to avoid
+ duplicating all of the nightly builder names into the gecko
+ migration mozharness configs. (Which would surely get out of date
+ very quickly).
+ 2) Arbitrary builders ("post_merge_builders"). These are additional
+ builders to trigger that aren't part of the nightly builder set.
+ Previous example: hg bundle generation builders.
+ """
+ dirs = self.query_abs_dirs()
+ branch = self.config["to_repo_url"].rstrip("/").split("/")[-1]
+ revision = self.query_to_revision()
+ # Horrible hack because our internal buildapi interface doesn't let us
+ # actually do anything. Need to use the public one w/ auth.
+ username = raw_input("LDAP Username: ")
+ password = getpass(prompt="LDAP Password: ")
+ auth = (username, password)
+ for builder in self.config["post_merge_builders"]:
+ self.trigger_arbitrary_job(builder, branch, revision, auth)
+ for nightly_branch in self.config["post_merge_nightly_branches"]:
+ nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch))
+ self.trigger_nightly_builds(nightly_branch, nightly_revision, auth)
+
+# __main__ {{{1
+if __name__ == '__main__':
+ GeckoMigration().run_and_exit()