diff options
Diffstat (limited to 'testing/mozharness/scripts/merge_day/gecko_migration.py')
| -rwxr-xr-x | testing/mozharness/scripts/merge_day/gecko_migration.py | 545 |
1 files changed, 545 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/merge_day/gecko_migration.py b/testing/mozharness/scripts/merge_day/gecko_migration.py new file mode 100755 index 000000000..7208630e0 --- /dev/null +++ b/testing/mozharness/scripts/merge_day/gecko_migration.py @@ -0,0 +1,545 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" gecko_migration.py + +Merge day script for gecko (mozilla-central -> mozilla-aurora, +mozilla-aurora -> mozilla-beta, mozilla-beta -> mozilla-release). + +Ported largely from +http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/beta2release.py +and +http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/merge_helper.py +""" + +import os +import pprint +import subprocess +import sys +from getpass import getpass + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.errors import HgErrorList +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.selfserve import SelfServeMixin +from mozharness.mozilla.updates.balrog import BalrogMixin +from mozharness.mozilla.buildbot import BuildbotMixin +from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin + +VALID_MIGRATION_BEHAVIORS = ( + "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr", + "bump_second_digit", +) + + +# GeckoMigration {{{1 +class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin, + SelfServeMixin, BuildbotMixin, + MercurialRepoManipulationMixin): + config_options = [ + [['--hg-user', ], { + "action": "store", + "dest": "hg_user", + "type": "string", + "default": "ffxbld <release@mozilla.com>", + "help": "Specify what user to use to commit to hg.", + }], + [['--balrog-api-root', ], { + "action": "store", + "dest": "balrog_api_root", + "type": "string", + "help": "Specify Balrog API root URL.", + }], + [['--balrog-username', ], { + "action": "store", + "dest": "balrog_username", + "type": "string", + "help": "Specify what user to connect to Balrog with.", + }], + [['--balrog-credentials-file', ], { + "action": "store", + "dest": "balrog_credentials_file", + "type": "string", + "help": "The file containing the Balrog credentials.", + }], + [['--remove-locale', ], { + "action": "extend", + "dest": "remove_locales", + "type": "string", + "help": "Comma separated list of locales to remove from the 'to' repo.", + }], + ] + gecko_repos = None + + def __init__(self, require_config_file=True): + super(GeckoMigration, self).__init__( + config_options=virtualenv_config_options + self.config_options, + all_actions=[ + 'clobber', + 'create-virtualenv', + 'clean-repos', + 'pull', + 'lock-update-paths', + 'migrate', + 'bump_second_digit', + 'commit-changes', + 'push', + 'trigger-builders', + ], + default_actions=[ + 'clean-repos', + 'pull', + 'migrate', + ], + require_config_file=require_config_file + ) + self.run_sanity_check() + +# Helper methods {{{1 + def run_sanity_check(self): + """ Verify the configs look sane before proceeding. + """ + message = "" + if self.config['migration_behavior'] not in VALID_MIGRATION_BEHAVIORS: + message += "%s must be one of %s!\n" % (self.config['migration_behavior'], VALID_MIGRATION_BEHAVIORS) + if self.config['migration_behavior'] == 'beta_to_release': + if self.config.get("require_remove_locales") and not self.config.get("remove_locales") and 'migrate' in self.actions: + message += "You must specify --remove-locale!\n" + else: + if self.config.get("require_remove_locales") or self.config.get("remove_locales"): + self.warning("--remove-locale isn't valid unless you're using beta_to_release migration_behavior!\n") + if message: + self.fatal(message) + + def query_abs_dirs(self): + """ Allow for abs_from_dir and abs_to_dir + """ + if self.abs_dirs: + return self.abs_dirs + dirs = super(GeckoMigration, self).query_abs_dirs() + self.abs_dirs['abs_tools_dir'] = os.path.join( + dirs['abs_work_dir'], 'tools' + ) + self.abs_dirs['abs_tools_lib_dir'] = os.path.join( + dirs['abs_work_dir'], 'tools', 'lib', 'python' + ) + for k in ('from', 'to'): + url = self.config.get("%s_repo_url" % k) + if url: + dir_name = self.get_filename_from_url(url) + self.info("adding %s" % dir_name) + self.abs_dirs['abs_%s_dir' % k] = os.path.join( + dirs['abs_work_dir'], dir_name + ) + return self.abs_dirs + + def query_repos(self): + """ Build a list of repos to clone. + """ + if self.gecko_repos: + return self.gecko_repos + self.info("Building gecko_repos list...") + dirs = self.query_abs_dirs() + self.gecko_repos = [] + for k in ('from', 'to'): + repo_key = "%s_repo_url" % k + url = self.config.get(repo_key) + if url: + self.gecko_repos.append({ + "repo": url, + "branch": self.config.get("%s_repo_branch" % (k,), "default"), + "dest": dirs['abs_%s_dir' % k], + "vcs": "hg", + # "hg" vcs uses robustcheckout extension requires the use of a share + # but having a share breaks migration logic when merging repos. + # Solution: tell hg vcs to create a unique share directory for each + # gecko repo. see mozharness/base/vcs/mercurial.py for implementation + "use_vcs_unique_share": True, + }) + else: + self.warning("Skipping %s" % repo_key) + self.info(pprint.pformat(self.gecko_repos)) + return self.gecko_repos + + def query_commit_dirs(self): + dirs = self.query_abs_dirs() + commit_dirs = [dirs['abs_to_dir']] + if self.config['migration_behavior'] == 'central_to_aurora': + commit_dirs.append(dirs['abs_from_dir']) + return commit_dirs + + def query_commit_message(self): + return "Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release" + + def query_push_dirs(self): + dirs = self.query_abs_dirs() + return dirs.get('abs_from_dir'), dirs.get('abs_to_dir') + + def query_push_args(self, cwd): + if cwd == self.query_abs_dirs()['abs_to_dir'] and \ + self.config['migration_behavior'] == 'beta_to_release': + return ['--new-branch', '-r', '.'] + else: + return ['-r', '.'] + + def query_from_revision(self): + """ Shortcut to get the revision for the from repo + """ + dirs = self.query_abs_dirs() + return self.query_hg_revision(dirs['abs_from_dir']) + + def query_to_revision(self): + """ Shortcut to get the revision for the to repo + """ + dirs = self.query_abs_dirs() + return self.query_hg_revision(dirs['abs_to_dir']) + + def hg_merge_via_debugsetparents(self, cwd, old_head, new_head, + preserve_tags=True, user=None): + """ Merge 2 heads avoiding non-fastforward commits + """ + hg = self.query_exe('hg', return_type='list') + cmd = hg + ['debugsetparents', new_head, old_head] + self.run_command(cmd, cwd=cwd, error_list=HgErrorList, + halt_on_failure=True) + self.hg_commit( + cwd, + message="Merge old head via |hg debugsetparents %s %s|. " + "CLOSED TREE DONTBUILD a=release" % (new_head, old_head), + user=user + ) + if preserve_tags: + # I don't know how to do this elegantly. + # I'm reverting .hgtags to old_head, then appending the new tags + # from new_head to .hgtags, and hoping nothing goes wrong. + # I'd rather not write patch files from scratch, so this seems + # like a slightly more complex but less objectionable method? + self.info("Trying to preserve tags from before debugsetparents...") + dirs = self.query_abs_dirs() + patch_file = os.path.join(dirs['abs_work_dir'], 'patch_file') + self.run_command( + subprocess.list2cmdline(hg + ['diff', '-r', old_head, '.hgtags', '-U9', '>', patch_file]), + cwd=cwd, + ) + self.run_command( + ['patch', '-R', '-p1', '-i', patch_file], + cwd=cwd, + halt_on_failure=True, + ) + tag_diff = self.read_from_file(patch_file) + with self.opened(os.path.join(cwd, '.hgtags'), open_mode='a') as (fh, err): + if err: + self.fatal("Can't append to .hgtags!") + for n, line in enumerate(tag_diff.splitlines()): + # The first 4 lines of a patch are headers, so we ignore them. + if n < 5: + continue + # Even after that, the only lines we really care about are + # additions to the file. + # TODO: why do we only care about additions? I couldn't + # figure that out by reading this code. + if not line.startswith('+'): + continue + line = line.replace('+', '') + (changeset, tag) = line.split(' ') + if len(changeset) != 40: + continue + fh.write("%s\n" % line) + out = self.get_output_from_command(['hg', 'status', '.hgtags'], + cwd=cwd) + if out: + self.hg_commit( + cwd, + message="Preserve old tags after debugsetparents. " + "CLOSED TREE DONTBUILD a=release", + user=user, + ) + else: + self.info(".hgtags file is identical, no need to commit") + + def remove_locales(self, file_name, locales): + """ Remove locales from shipped-locales (m-r only) + """ + contents = self.read_from_file(file_name) + new_contents = "" + for line in contents.splitlines(): + locale = line.split()[0] + if locale not in locales: + new_contents += "%s\n" % line + else: + self.info("Removed locale: %s" % locale) + self.write_to_file(file_name, new_contents) + + def touch_clobber_file(self, cwd): + clobber_file = os.path.join(cwd, 'CLOBBER') + contents = self.read_from_file(clobber_file) + new_contents = "" + for line in contents.splitlines(): + line = line.strip() + if line.startswith("#") or line == '': + new_contents += "%s\n" % line + new_contents += "Merge day clobber" + self.write_to_file(clobber_file, new_contents) + + def bump_version(self, cwd, curr_version, next_version, curr_suffix, + next_suffix, bump_major=False): + """ Bump versions (m-c, m-a, m-b). + + At some point we may want to unhardcode these filenames into config + """ + curr_weave_version = str(int(curr_version) + 2) + next_weave_version = str(int(curr_weave_version) + 1) + for f in self.config["version_files"]: + from_ = "%s.0%s" % (curr_version, curr_suffix) + to = "%s.0%s%s" % (next_version, next_suffix, f["suffix"]) + self.replace(os.path.join(cwd, f["file"]), from_, to) + + # only applicable for m-c + if bump_major: + self.replace( + os.path.join(cwd, "xpcom/components/Module.h"), + "static const unsigned int kVersion = %s;" % curr_version, + "static const unsigned int kVersion = %s;" % next_version + ) + self.replace( + os.path.join(cwd, "services/sync/moz.build"), + "DEFINES['weave_version'] = '1.%s.0'" % curr_weave_version, + "DEFINES['weave_version'] = '1.%s.0'" % next_weave_version + ) + + # Branch-specific workflow helper methods {{{1 + def central_to_aurora(self, end_tag): + """ mozilla-central -> mozilla-aurora behavior. + + We could have all of these individually toggled by flags, but + by separating into workflow methods we can be more precise about + what happens in each workflow, while allowing for things like + staging beta user repo migrations. + """ + dirs = self.query_abs_dirs() + self.info("Reverting locales") + hg = self.query_exe("hg", return_type="list") + for f in self.config["locale_files"]: + self.run_command( + hg + ["revert", "-r", end_tag, f], + cwd=dirs['abs_to_dir'], + error_list=HgErrorList, + halt_on_failure=True, + ) + next_ma_version = self.get_version(dirs['abs_to_dir'])[0] + self.bump_version(dirs['abs_to_dir'], next_ma_version, next_ma_version, "a1", "a2") + self.apply_replacements() + # bump m-c version + curr_mc_version = self.get_version(dirs['abs_from_dir'])[0] + next_mc_version = str(int(curr_mc_version) + 1) + self.bump_version( + dirs['abs_from_dir'], curr_mc_version, next_mc_version, "a1", "a1", + bump_major=True + ) + # touch clobber files + self.touch_clobber_file(dirs['abs_from_dir']) + self.touch_clobber_file(dirs['abs_to_dir']) + + def aurora_to_beta(self, *args, **kwargs): + """ mozilla-aurora -> mozilla-beta behavior. + + We could have all of these individually toggled by flags, but + by separating into workflow methods we can be more precise about + what happens in each workflow, while allowing for things like + staging beta user repo migrations. + """ + dirs = self.query_abs_dirs() + mb_version = self.get_version(dirs['abs_to_dir'])[0] + self.bump_version(dirs['abs_to_dir'], mb_version, mb_version, "a2", "") + self.apply_replacements() + self.touch_clobber_file(dirs['abs_to_dir']) + # TODO mozconfig diffing + # The build/tools version only checks the mozconfigs from hgweb, so + # can't help pre-push. The in-tree mozconfig diffing requires a mach + # virtualenv to be installed. If we want this sooner we can put this + # in the push action; otherwise we may just wait until we have in-tree + # mozconfig checking. + + def beta_to_release(self, *args, **kwargs): + """ mozilla-beta -> mozilla-release behavior. + + We could have all of these individually toggled by flags, but + by separating into workflow methods we can be more precise about + what happens in each workflow, while allowing for things like + staging beta user repo migrations. + """ + dirs = self.query_abs_dirs() + # Reset display_version.txt + for f in self.config["copy_files"]: + self.copyfile( + os.path.join(dirs['abs_to_dir'], f["src"]), + os.path.join(dirs['abs_to_dir'], f["dst"])) + + self.apply_replacements() + if self.config.get("remove_locales"): + self.remove_locales( + os.path.join(dirs['abs_to_dir'], "browser/locales/shipped-locales"), + self.config['remove_locales'] + ) + self.touch_clobber_file(dirs['abs_to_dir']) + + def release_to_esr(self, *args, **kwargs): + """ mozilla-release -> mozilla-esrNN behavior. """ + dirs = self.query_abs_dirs() + for to_transplant in self.config.get("transplant_patches", []): + self.transplant(repo=to_transplant["repo"], + changeset=to_transplant["changeset"], + cwd=dirs['abs_to_dir']) + self.apply_replacements() + self.touch_clobber_file(dirs['abs_to_dir']) + + def apply_replacements(self): + dirs = self.query_abs_dirs() + for f, from_, to in self.config["replacements"]: + self.replace(os.path.join(dirs['abs_to_dir'], f), from_, to) + + def transplant(self, repo, changeset, cwd): + """Transplant a Mercurial changeset from a remote repository.""" + hg = self.query_exe("hg", return_type="list") + cmd = hg + ["--config", "extensions.transplant=", "transplant", + "--source", repo, changeset] + self.info("Transplanting %s from %s" % (changeset, repo)) + status = self.run_command( + cmd, + cwd=cwd, + error_list=HgErrorList, + ) + if status != 0: + self.fatal("Cannot transplant %s from %s properly" % + (changeset, repo)) + + def pull_from_repo(self, from_dir, to_dir, revision=None, branch=None): + """ Pull from one repo to another. """ + hg = self.query_exe("hg", return_type="list") + cmd = hg + ["pull"] + if revision: + cmd.extend(["-r", revision]) + cmd.append(from_dir) + self.run_command( + cmd, + cwd=to_dir, + error_list=HgErrorList, + halt_on_failure=True, + ) + cmd = hg + ["update", "-C"] + if branch or revision: + cmd.extend(["-r", branch or revision]) + self.run_command( + cmd, + cwd=to_dir, + error_list=HgErrorList, + halt_on_failure=True, + ) + +# Actions {{{1 + def bump_second_digit(self, *args, **kwargs): + """Bump second digit. + + ESR need only the second digit bumped as a part of merge day.""" + dirs = self.query_abs_dirs() + version = self.get_version(dirs['abs_to_dir']) + curr_version = ".".join(version) + next_version = list(version) + # bump the second digit + next_version[1] = str(int(next_version[1]) + 1) + # Take major+minor and append '0' accordng to Firefox version schema. + # 52.0 will become 52.1.0, not 52.1 + next_version = ".".join(next_version[:2] + ['0']) + for f in self.config["version_files"]: + self.replace(os.path.join(dirs['abs_to_dir'], f["file"]), + curr_version, next_version + f["suffix"]) + self.touch_clobber_file(dirs['abs_to_dir']) + + def pull(self): + """ Pull tools first, then clone the gecko repos + """ + repos = [{ + "repo": self.config["tools_repo_url"], + "branch": self.config["tools_repo_branch"], + "dest": "tools", + "vcs": "hg", + }] + self.query_repos() + super(GeckoMigration, self).pull(repos=repos) + + def lock_update_paths(self): + self.lock_balrog_rules(self.config["balrog_rules_to_lock"]) + + def migrate(self): + """ Perform the migration. + """ + dirs = self.query_abs_dirs() + from_fx_major_version = self.get_version(dirs['abs_from_dir'])[0] + to_fx_major_version = self.get_version(dirs['abs_to_dir'])[0] + base_from_rev = self.query_from_revision() + base_to_rev = self.query_to_revision() + base_tag = self.config['base_tag'] % {'major_version': from_fx_major_version} + end_tag = self.config['end_tag'] % {'major_version': to_fx_major_version} + self.hg_tag( + dirs['abs_from_dir'], base_tag, user=self.config['hg_user'], + revision=base_from_rev, + ) + new_from_rev = self.query_from_revision() + self.info("New revision %s" % new_from_rev) + pull_revision = None + if not self.config.get("pull_all_branches"): + pull_revision = new_from_rev + self.pull_from_repo( + dirs['abs_from_dir'], dirs['abs_to_dir'], + revision=pull_revision, + branch="default", + ) + if self.config.get("requires_head_merge") is not False: + self.hg_merge_via_debugsetparents( + dirs['abs_to_dir'], old_head=base_to_rev, new_head=new_from_rev, + user=self.config['hg_user'], + ) + self.hg_tag( + dirs['abs_to_dir'], end_tag, user=self.config['hg_user'], + revision=base_to_rev, force=True, + ) + # Call beta_to_release etc. + if not hasattr(self, self.config['migration_behavior']): + self.fatal("Don't know how to proceed with migration_behavior %s !" % self.config['migration_behavior']) + getattr(self, self.config['migration_behavior'])(end_tag=end_tag) + self.info("Verify the diff, and apply any manual changes, such as disabling features, and --commit-changes") + + def trigger_builders(self): + """Triggers builders that should be run directly after a merge. + There are two different types of things we trigger: + 1) Nightly builds ("post_merge_nightly_branches" in the config). + These are triggered with buildapi's nightly build endpoint to avoid + duplicating all of the nightly builder names into the gecko + migration mozharness configs. (Which would surely get out of date + very quickly). + 2) Arbitrary builders ("post_merge_builders"). These are additional + builders to trigger that aren't part of the nightly builder set. + Previous example: hg bundle generation builders. + """ + dirs = self.query_abs_dirs() + branch = self.config["to_repo_url"].rstrip("/").split("/")[-1] + revision = self.query_to_revision() + # Horrible hack because our internal buildapi interface doesn't let us + # actually do anything. Need to use the public one w/ auth. + username = raw_input("LDAP Username: ") + password = getpass(prompt="LDAP Password: ") + auth = (username, password) + for builder in self.config["post_merge_builders"]: + self.trigger_arbitrary_job(builder, branch, revision, auth) + for nightly_branch in self.config["post_merge_nightly_branches"]: + nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch)) + self.trigger_nightly_builds(nightly_branch, nightly_revision, auth) + +# __main__ {{{1 +if __name__ == '__main__': + GeckoMigration().run_and_exit() |