Add m-esr52 at 52.6.0

1 files changed, 545 insertions, 0 deletions

diff --git a/testing/mozharness/scripts/merge_day/gecko_migration.py b/testing/mozharness/scripts/merge_day/gecko_migration.py
new file mode 100755
index 000000000..7208630e0
--- /dev/null
+++ b/testing/mozharness/scripts/merge_day/gecko_migration.py
@@ -0,0 +1,545 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" gecko_migration.py
+
+Merge day script for gecko (mozilla-central -> mozilla-aurora,
+mozilla-aurora -> mozilla-beta, mozilla-beta -> mozilla-release).
+
+Ported largely from
+http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/beta2release.py
+and
+http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/merge_helper.py
+"""
+
+import os
+import pprint
+import subprocess
+import sys
+from getpass import getpass
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.errors import HgErrorList
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.selfserve import SelfServeMixin
+from mozharness.mozilla.updates.balrog import BalrogMixin
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
+
+VALID_MIGRATION_BEHAVIORS = (
+    "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr",
+    "bump_second_digit",
+)
+
+
+# GeckoMigration {{{1
+class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin,
+                     SelfServeMixin, BuildbotMixin,
+                     MercurialRepoManipulationMixin):
+    config_options = [
+        [['--hg-user', ], {
+            "action": "store",
+            "dest": "hg_user",
+            "type": "string",
+            "default": "ffxbld <release@mozilla.com>",
+            "help": "Specify what user to use to commit to hg.",
+        }],
+        [['--balrog-api-root', ], {
+            "action": "store",
+            "dest": "balrog_api_root",
+            "type": "string",
+            "help": "Specify Balrog API root URL.",
+        }],
+        [['--balrog-username', ], {
+            "action": "store",
+            "dest": "balrog_username",
+            "type": "string",
+            "help": "Specify what user to connect to Balrog with.",
+        }],
+        [['--balrog-credentials-file', ], {
+            "action": "store",
+            "dest": "balrog_credentials_file",
+            "type": "string",
+            "help": "The file containing the Balrog credentials.",
+        }],
+        [['--remove-locale', ], {
+            "action": "extend",
+            "dest": "remove_locales",
+            "type": "string",
+            "help": "Comma separated list of locales to remove from the 'to' repo.",
+        }],
+    ]
+    gecko_repos = None
+
+    def __init__(self, require_config_file=True):
+        super(GeckoMigration, self).__init__(
+            config_options=virtualenv_config_options + self.config_options,
+            all_actions=[
+                'clobber',
+                'create-virtualenv',
+                'clean-repos',
+                'pull',
+                'lock-update-paths',
+                'migrate',
+                'bump_second_digit',
+                'commit-changes',
+                'push',
+                'trigger-builders',
+            ],
+            default_actions=[
+                'clean-repos',
+                'pull',
+                'migrate',
+            ],
+            require_config_file=require_config_file
+        )
+        self.run_sanity_check()
+
+# Helper methods {{{1
+    def run_sanity_check(self):
+        """ Verify the configs look sane before proceeding.
+            """
+        message = ""
+        if self.config['migration_behavior'] not in VALID_MIGRATION_BEHAVIORS:
+            message += "%s must be one of %s!\n" % (self.config['migration_behavior'], VALID_MIGRATION_BEHAVIORS)
+        if self.config['migration_behavior'] == 'beta_to_release':
+            if self.config.get("require_remove_locales") and not self.config.get("remove_locales") and 'migrate' in self.actions:
+                message += "You must specify --remove-locale!\n"
+        else:
+            if self.config.get("require_remove_locales") or self.config.get("remove_locales"):
+                self.warning("--remove-locale isn't valid unless you're using beta_to_release migration_behavior!\n")
+        if message:
+            self.fatal(message)
+
+    def query_abs_dirs(self):
+        """ Allow for abs_from_dir and abs_to_dir
+            """
+        if self.abs_dirs:
+            return self.abs_dirs
+        dirs = super(GeckoMigration, self).query_abs_dirs()
+        self.abs_dirs['abs_tools_dir'] = os.path.join(
+            dirs['abs_work_dir'], 'tools'
+        )
+        self.abs_dirs['abs_tools_lib_dir'] = os.path.join(
+            dirs['abs_work_dir'], 'tools', 'lib', 'python'
+        )
+        for k in ('from', 'to'):
+            url = self.config.get("%s_repo_url" % k)
+            if url:
+                dir_name = self.get_filename_from_url(url)
+                self.info("adding %s" % dir_name)
+                self.abs_dirs['abs_%s_dir' % k] = os.path.join(
+                    dirs['abs_work_dir'], dir_name
+                )
+        return self.abs_dirs
+
+    def query_repos(self):
+        """ Build a list of repos to clone.
+            """
+        if self.gecko_repos:
+            return self.gecko_repos
+        self.info("Building gecko_repos list...")
+        dirs = self.query_abs_dirs()
+        self.gecko_repos = []
+        for k in ('from', 'to'):
+            repo_key = "%s_repo_url" % k
+            url = self.config.get(repo_key)
+            if url:
+                self.gecko_repos.append({
+                    "repo": url,
+                    "branch": self.config.get("%s_repo_branch" % (k,), "default"),
+                    "dest": dirs['abs_%s_dir' % k],
+                    "vcs": "hg",
+                    # "hg" vcs uses robustcheckout extension requires the use of a share
+                    # but having a share breaks migration logic when merging repos.
+                    # Solution: tell hg vcs to create a unique share directory for each
+                    # gecko repo. see mozharness/base/vcs/mercurial.py for implementation
+                    "use_vcs_unique_share": True,
+                })
+            else:
+                self.warning("Skipping %s" % repo_key)
+        self.info(pprint.pformat(self.gecko_repos))
+        return self.gecko_repos
+
+    def query_commit_dirs(self):
+        dirs = self.query_abs_dirs()
+        commit_dirs = [dirs['abs_to_dir']]
+        if self.config['migration_behavior'] == 'central_to_aurora':
+            commit_dirs.append(dirs['abs_from_dir'])
+        return commit_dirs
+
+    def query_commit_message(self):
+        return "Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release"
+
+    def query_push_dirs(self):
+        dirs = self.query_abs_dirs()
+        return dirs.get('abs_from_dir'), dirs.get('abs_to_dir')
+
+    def query_push_args(self, cwd):
+        if cwd == self.query_abs_dirs()['abs_to_dir'] and \
+                self.config['migration_behavior'] == 'beta_to_release':
+            return ['--new-branch', '-r', '.']
+        else:
+            return ['-r', '.']
+
+    def query_from_revision(self):
+        """ Shortcut to get the revision for the from repo
+            """
+        dirs = self.query_abs_dirs()
+        return self.query_hg_revision(dirs['abs_from_dir'])
+
+    def query_to_revision(self):
+        """ Shortcut to get the revision for the to repo
+            """
+        dirs = self.query_abs_dirs()
+        return self.query_hg_revision(dirs['abs_to_dir'])
+
+    def hg_merge_via_debugsetparents(self, cwd, old_head, new_head,
+                                     preserve_tags=True, user=None):
+        """ Merge 2 heads avoiding non-fastforward commits
+            """
+        hg = self.query_exe('hg', return_type='list')
+        cmd = hg + ['debugsetparents', new_head, old_head]
+        self.run_command(cmd, cwd=cwd, error_list=HgErrorList,
+                         halt_on_failure=True)
+        self.hg_commit(
+            cwd,
+            message="Merge old head via |hg debugsetparents %s %s|. "
+            "CLOSED TREE DONTBUILD a=release" % (new_head, old_head),
+            user=user
+        )
+        if preserve_tags:
+            # I don't know how to do this elegantly.
+            # I'm reverting .hgtags to old_head, then appending the new tags
+            # from new_head to .hgtags, and hoping nothing goes wrong.
+            # I'd rather not write patch files from scratch, so this seems
+            # like a slightly more complex but less objectionable method?
+            self.info("Trying to preserve tags from before debugsetparents...")
+            dirs = self.query_abs_dirs()
+            patch_file = os.path.join(dirs['abs_work_dir'], 'patch_file')
+            self.run_command(
+                subprocess.list2cmdline(hg + ['diff', '-r', old_head, '.hgtags', '-U9', '>', patch_file]),
+                cwd=cwd,
+            )
+            self.run_command(
+                ['patch', '-R', '-p1', '-i', patch_file],
+                cwd=cwd,
+                halt_on_failure=True,
+            )
+            tag_diff = self.read_from_file(patch_file)
+            with self.opened(os.path.join(cwd, '.hgtags'), open_mode='a') as (fh, err):
+                if err:
+                    self.fatal("Can't append to .hgtags!")
+                for n, line in enumerate(tag_diff.splitlines()):
+                    # The first 4 lines of a patch are headers, so we ignore them.
+                    if n < 5:
+                        continue
+                    # Even after that, the only lines we really care about are
+                    # additions to the file.
+                    # TODO: why do we only care about additions? I couldn't
+                    # figure that out by reading this code.
+                    if not line.startswith('+'):
+                        continue
+                    line = line.replace('+', '')
+                    (changeset, tag) = line.split(' ')
+                    if len(changeset) != 40:
+                        continue
+                    fh.write("%s\n" % line)
+            out = self.get_output_from_command(['hg', 'status', '.hgtags'],
+                                               cwd=cwd)
+            if out:
+                self.hg_commit(
+                    cwd,
+                    message="Preserve old tags after debugsetparents. "
+                    "CLOSED TREE DONTBUILD a=release",
+                    user=user,
+                )
+            else:
+                self.info(".hgtags file is identical, no need to commit")
+
+    def remove_locales(self, file_name, locales):
+        """ Remove locales from shipped-locales (m-r only)
+            """
+        contents = self.read_from_file(file_name)
+        new_contents = ""
+        for line in contents.splitlines():
+            locale = line.split()[0]
+            if locale not in locales:
+                new_contents += "%s\n" % line
+            else:
+                self.info("Removed locale: %s" % locale)
+        self.write_to_file(file_name, new_contents)
+
+    def touch_clobber_file(self, cwd):
+        clobber_file = os.path.join(cwd, 'CLOBBER')
+        contents = self.read_from_file(clobber_file)
+        new_contents = ""
+        for line in contents.splitlines():
+            line = line.strip()
+            if line.startswith("#") or line == '':
+                new_contents += "%s\n" % line
+        new_contents += "Merge day clobber"
+        self.write_to_file(clobber_file, new_contents)
+
+    def bump_version(self, cwd, curr_version, next_version, curr_suffix,
+                     next_suffix, bump_major=False):
+        """ Bump versions (m-c, m-a, m-b).
+
+            At some point we may want to unhardcode these filenames into config
+            """
+        curr_weave_version = str(int(curr_version) + 2)
+        next_weave_version = str(int(curr_weave_version) + 1)
+        for f in self.config["version_files"]:
+            from_ = "%s.0%s" % (curr_version, curr_suffix)
+            to = "%s.0%s%s" % (next_version, next_suffix, f["suffix"])
+            self.replace(os.path.join(cwd, f["file"]), from_, to)
+
+        # only applicable for m-c
+        if bump_major:
+            self.replace(
+                os.path.join(cwd, "xpcom/components/Module.h"),
+                "static const unsigned int kVersion = %s;" % curr_version,
+                "static const unsigned int kVersion = %s;" % next_version
+            )
+            self.replace(
+                os.path.join(cwd, "services/sync/moz.build"),
+                "DEFINES['weave_version'] = '1.%s.0'" % curr_weave_version,
+                "DEFINES['weave_version'] = '1.%s.0'" % next_weave_version
+            )
+
+    # Branch-specific workflow helper methods {{{1
+    def central_to_aurora(self, end_tag):
+        """ mozilla-central -> mozilla-aurora behavior.
+
+            We could have all of these individually toggled by flags, but
+            by separating into workflow methods we can be more precise about
+            what happens in each workflow, while allowing for things like
+            staging beta user repo migrations.
+            """
+        dirs = self.query_abs_dirs()
+        self.info("Reverting locales")
+        hg = self.query_exe("hg", return_type="list")
+        for f in self.config["locale_files"]:
+            self.run_command(
+                hg + ["revert", "-r", end_tag, f],
+                cwd=dirs['abs_to_dir'],
+                error_list=HgErrorList,
+                halt_on_failure=True,
+            )
+        next_ma_version = self.get_version(dirs['abs_to_dir'])[0]
+        self.bump_version(dirs['abs_to_dir'], next_ma_version, next_ma_version, "a1", "a2")
+        self.apply_replacements()
+        # bump m-c version
+        curr_mc_version = self.get_version(dirs['abs_from_dir'])[0]
+        next_mc_version = str(int(curr_mc_version) + 1)
+        self.bump_version(
+            dirs['abs_from_dir'], curr_mc_version, next_mc_version, "a1", "a1",
+            bump_major=True
+        )
+        # touch clobber files
+        self.touch_clobber_file(dirs['abs_from_dir'])
+        self.touch_clobber_file(dirs['abs_to_dir'])
+
+    def aurora_to_beta(self, *args, **kwargs):
+        """ mozilla-aurora -> mozilla-beta behavior.
+
+            We could have all of these individually toggled by flags, but
+            by separating into workflow methods we can be more precise about
+            what happens in each workflow, while allowing for things like
+            staging beta user repo migrations.
+            """
+        dirs = self.query_abs_dirs()
+        mb_version = self.get_version(dirs['abs_to_dir'])[0]
+        self.bump_version(dirs['abs_to_dir'], mb_version, mb_version, "a2", "")
+        self.apply_replacements()
+        self.touch_clobber_file(dirs['abs_to_dir'])
+        # TODO mozconfig diffing
+        # The build/tools version only checks the mozconfigs from hgweb, so
+        # can't help pre-push.  The in-tree mozconfig diffing requires a mach
+        # virtualenv to be installed.  If we want this sooner we can put this
+        # in the push action; otherwise we may just wait until we have in-tree
+        # mozconfig checking.
+
+    def beta_to_release(self, *args, **kwargs):
+        """ mozilla-beta -> mozilla-release behavior.
+
+            We could have all of these individually toggled by flags, but
+            by separating into workflow methods we can be more precise about
+            what happens in each workflow, while allowing for things like
+            staging beta user repo migrations.
+            """
+        dirs = self.query_abs_dirs()
+        # Reset display_version.txt
+        for f in self.config["copy_files"]:
+            self.copyfile(
+                os.path.join(dirs['abs_to_dir'], f["src"]),
+                os.path.join(dirs['abs_to_dir'], f["dst"]))
+
+        self.apply_replacements()
+        if self.config.get("remove_locales"):
+            self.remove_locales(
+                os.path.join(dirs['abs_to_dir'], "browser/locales/shipped-locales"),
+                self.config['remove_locales']
+            )
+        self.touch_clobber_file(dirs['abs_to_dir'])
+
+    def release_to_esr(self, *args, **kwargs):
+        """ mozilla-release -> mozilla-esrNN behavior. """
+        dirs = self.query_abs_dirs()
+        for to_transplant in self.config.get("transplant_patches", []):
+            self.transplant(repo=to_transplant["repo"],
+                            changeset=to_transplant["changeset"],
+                            cwd=dirs['abs_to_dir'])
+        self.apply_replacements()
+        self.touch_clobber_file(dirs['abs_to_dir'])
+
+    def apply_replacements(self):
+        dirs = self.query_abs_dirs()
+        for f, from_, to in self.config["replacements"]:
+            self.replace(os.path.join(dirs['abs_to_dir'], f), from_, to)
+
+    def transplant(self, repo, changeset, cwd):
+        """Transplant a Mercurial changeset from a remote repository."""
+        hg = self.query_exe("hg", return_type="list")
+        cmd = hg + ["--config", "extensions.transplant=", "transplant",
+                    "--source", repo, changeset]
+        self.info("Transplanting %s from %s" % (changeset, repo))
+        status = self.run_command(
+            cmd,
+            cwd=cwd,
+            error_list=HgErrorList,
+        )
+        if status != 0:
+            self.fatal("Cannot transplant %s from %s properly" %
+                       (changeset, repo))
+
+    def pull_from_repo(self, from_dir, to_dir, revision=None, branch=None):
+        """ Pull from one repo to another. """
+        hg = self.query_exe("hg", return_type="list")
+        cmd = hg + ["pull"]
+        if revision:
+            cmd.extend(["-r", revision])
+        cmd.append(from_dir)
+        self.run_command(
+            cmd,
+            cwd=to_dir,
+            error_list=HgErrorList,
+            halt_on_failure=True,
+        )
+        cmd = hg + ["update", "-C"]
+        if branch or revision:
+            cmd.extend(["-r", branch or revision])
+        self.run_command(
+            cmd,
+            cwd=to_dir,
+            error_list=HgErrorList,
+            halt_on_failure=True,
+        )
+
+# Actions {{{1
+    def bump_second_digit(self, *args, **kwargs):
+        """Bump second digit.
+
+         ESR need only the second digit bumped as a part of merge day."""
+        dirs = self.query_abs_dirs()
+        version = self.get_version(dirs['abs_to_dir'])
+        curr_version = ".".join(version)
+        next_version = list(version)
+        # bump the second digit
+        next_version[1] = str(int(next_version[1]) + 1)
+        # Take major+minor and append '0' accordng to Firefox version schema.
+        # 52.0 will become 52.1.0, not 52.1
+        next_version = ".".join(next_version[:2] + ['0'])
+        for f in self.config["version_files"]:
+            self.replace(os.path.join(dirs['abs_to_dir'], f["file"]),
+                         curr_version, next_version + f["suffix"])
+        self.touch_clobber_file(dirs['abs_to_dir'])
+
+    def pull(self):
+        """ Pull tools first, then clone the gecko repos
+            """
+        repos = [{
+            "repo": self.config["tools_repo_url"],
+            "branch": self.config["tools_repo_branch"],
+            "dest": "tools",
+            "vcs": "hg",
+        }] + self.query_repos()
+        super(GeckoMigration, self).pull(repos=repos)
+
+    def lock_update_paths(self):
+        self.lock_balrog_rules(self.config["balrog_rules_to_lock"])
+
+    def migrate(self):
+        """ Perform the migration.
+            """
+        dirs = self.query_abs_dirs()
+        from_fx_major_version = self.get_version(dirs['abs_from_dir'])[0]
+        to_fx_major_version = self.get_version(dirs['abs_to_dir'])[0]
+        base_from_rev = self.query_from_revision()
+        base_to_rev = self.query_to_revision()
+        base_tag = self.config['base_tag'] % {'major_version': from_fx_major_version}
+        end_tag = self.config['end_tag'] % {'major_version': to_fx_major_version}
+        self.hg_tag(
+            dirs['abs_from_dir'], base_tag, user=self.config['hg_user'],
+            revision=base_from_rev,
+        )
+        new_from_rev = self.query_from_revision()
+        self.info("New revision %s" % new_from_rev)
+        pull_revision = None
+        if not self.config.get("pull_all_branches"):
+            pull_revision = new_from_rev
+        self.pull_from_repo(
+            dirs['abs_from_dir'], dirs['abs_to_dir'],
+            revision=pull_revision,
+            branch="default",
+        )
+        if self.config.get("requires_head_merge") is not False:
+            self.hg_merge_via_debugsetparents(
+                dirs['abs_to_dir'], old_head=base_to_rev, new_head=new_from_rev,
+                user=self.config['hg_user'],
+            )
+        self.hg_tag(
+            dirs['abs_to_dir'], end_tag, user=self.config['hg_user'],
+            revision=base_to_rev, force=True,
+        )
+        # Call beta_to_release etc.
+        if not hasattr(self, self.config['migration_behavior']):
+            self.fatal("Don't know how to proceed with migration_behavior %s !" % self.config['migration_behavior'])
+        getattr(self, self.config['migration_behavior'])(end_tag=end_tag)
+        self.info("Verify the diff, and apply any manual changes, such as disabling features, and --commit-changes")
+
+    def trigger_builders(self):
+        """Triggers builders that should be run directly after a merge.
+        There are two different types of things we trigger:
+        1) Nightly builds ("post_merge_nightly_branches" in the config).
+           These are triggered with buildapi's nightly build endpoint to avoid
+           duplicating all of the nightly builder names into the gecko
+           migration mozharness configs. (Which would surely get out of date
+           very quickly).
+        2) Arbitrary builders ("post_merge_builders"). These are additional
+           builders to trigger that aren't part of the nightly builder set.
+           Previous example: hg bundle generation builders.
+        """
+        dirs = self.query_abs_dirs()
+        branch = self.config["to_repo_url"].rstrip("/").split("/")[-1]
+        revision = self.query_to_revision()
+        # Horrible hack because our internal buildapi interface doesn't let us
+        # actually do anything. Need to use the public one w/ auth.
+        username = raw_input("LDAP Username: ")
+        password = getpass(prompt="LDAP Password: ")
+        auth = (username, password)
+        for builder in self.config["post_merge_builders"]:
+            self.trigger_arbitrary_job(builder, branch, revision, auth)
+        for nightly_branch in self.config["post_merge_nightly_branches"]:
+            nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch))
+            self.trigger_nightly_builds(nightly_branch, nightly_revision, auth)
+
+# __main__ {{{1
+if __name__ == '__main__':
+    GeckoMigration().run_and_exit()