path: root/testing/mozharness/scripts/merge_day/gecko_migration.py

#!/usr/bin/env python
# lint_ignore=E501
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
""" gecko_migration.py

Merge day script for gecko (mozilla-central -> mozilla-aurora,
mozilla-aurora -> mozilla-beta, mozilla-beta -> mozilla-release).

Ported largely from
http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/beta2release.py
and
http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/merge_helper.py
"""

import os
import pprint
import subprocess
import sys
from getpass import getpass

sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))

from mozharness.base.errors import HgErrorList
from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.selfserve import SelfServeMixin
from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.mozilla.buildbot import BuildbotMixin
from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin

VALID_MIGRATION_BEHAVIORS = (
    "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr",
    "bump_second_digit",
)


# GeckoMigration {{{1
class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin,
                     SelfServeMixin, BuildbotMixin,
                     MercurialRepoManipulationMixin):
    config_options = [
        [['--hg-user', ], {
            "action": "store",
            "dest": "hg_user",
            "type": "string",
            "default": "ffxbld <release@mozilla.com>",
            "help": "Specify what user to use to commit to hg.",
        }],
        [['--balrog-api-root', ], {
            "action": "store",
            "dest": "balrog_api_root",
            "type": "string",
            "help": "Specify Balrog API root URL.",
        }],
        [['--balrog-username', ], {
            "action": "store",
            "dest": "balrog_username",
            "type": "string",
            "help": "Specify what user to connect to Balrog with.",
        }],
        [['--balrog-credentials-file', ], {
            "action": "store",
            "dest": "balrog_credentials_file",
            "type": "string",
            "help": "The file containing the Balrog credentials.",
        }],
        [['--remove-locale', ], {
            "action": "extend",
            "dest": "remove_locales",
            "type": "string",
            "help": "Comma separated list of locales to remove from the 'to' repo.",
        }],
    ]
    gecko_repos = None

    def __init__(self, require_config_file=True):
        super(GeckoMigration, self).__init__(
            config_options=virtualenv_config_options + self.config_options,
            all_actions=[
                'clobber',
                'create-virtualenv',
                'clean-repos',
                'pull',
                'lock-update-paths',
                'migrate',
                'bump_second_digit',
                'commit-changes',
                'push',
                'trigger-builders',
            ],
            default_actions=[
                'clean-repos',
                'pull',
                'migrate',
            ],
            require_config_file=require_config_file
        )
        self.run_sanity_check()

# Helper methods {{{1
    def run_sanity_check(self):
        """ Verify the configs look sane before proceeding.
            """
        message = ""
        if self.config['migration_behavior'] not in VALID_MIGRATION_BEHAVIORS:
            message += "%s must be one of %s!\n" % (self.config['migration_behavior'], VALID_MIGRATION_BEHAVIORS)
        if self.config['migration_behavior'] == 'beta_to_release':
            if self.config.get("require_remove_locales") and not self.config.get("remove_locales") and 'migrate' in self.actions:
                message += "You must specify --remove-locale!\n"
        else:
            if self.config.get("require_remove_locales") or self.config.get("remove_locales"):
                self.warning("--remove-locale isn't valid unless you're using beta_to_release migration_behavior!\n")
        if message:
            self.fatal(message)

    def query_abs_dirs(self):
        """ Allow for abs_from_dir and abs_to_dir
            """
        if self.abs_dirs:
            return self.abs_dirs
        dirs = super(GeckoMigration, self).query_abs_dirs()
        self.abs_dirs['abs_tools_dir'] = os.path.join(
            dirs['abs_work_dir'], 'tools'
        )
        self.abs_dirs['abs_tools_lib_dir'] = os.path.join(
            dirs['abs_work_dir'], 'tools', 'lib', 'python'
        )
        for k in ('from', 'to'):
            url = self.config.get("%s_repo_url" % k)
            if url:
                dir_name = self.get_filename_from_url(url)
                self.info("adding %s" % dir_name)
                self.abs_dirs['abs_%s_dir' % k] = os.path.join(
                    dirs['abs_work_dir'], dir_name
                )
        return self.abs_dirs

    def query_repos(self):
        """ Build a list of repos to clone.
            """
        if self.gecko_repos:
            return self.gecko_repos
        self.info("Building gecko_repos list...")
        dirs = self.query_abs_dirs()
        self.gecko_repos = []
        for k in ('from', 'to'):
            repo_key = "%s_repo_url" % k
            url = self.config.get(repo_key)
            if url:
                self.gecko_repos.append({
                    "repo": url,
                    "branch": self.config.get("%s_repo_branch" % (k,), "default"),
                    "dest": dirs['abs_%s_dir' % k],
                    "vcs": "hg",
                    # "hg" vcs uses robustcheckout extension requires the use of a share
                    # but having a share breaks migration logic when merging repos.
                    # Solution: tell hg vcs to create a unique share directory for each
                    # gecko repo. see mozharness/base/vcs/mercurial.py for implementation
                    "use_vcs_unique_share": True,
                })
            else:
                self.warning("Skipping %s" % repo_key)
        self.info(pprint.pformat(self.gecko_repos))
        return self.gecko_repos

    def query_commit_dirs(self):
        dirs = self.query_abs_dirs()
        commit_dirs = [dirs['abs_to_dir']]
        if self.config['migration_behavior'] == 'central_to_aurora':
            commit_dirs.append(dirs['abs_from_dir'])
        return commit_dirs

    def query_commit_message(self):
        return "Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release"

    def query_push_dirs(self):
        dirs = self.query_abs_dirs()
        return dirs.get('abs_from_dir'), dirs.get('abs_to_dir')

    def query_push_args(self, cwd):
        if cwd == self.query_abs_dirs()['abs_to_dir'] and \
                self.config['migration_behavior'] == 'beta_to_release':
            return ['--new-branch', '-r', '.']
        else:
            return ['-r', '.']

    def query_from_revision(self):
        """ Shortcut to get the revision for the from repo
            """
        dirs = self.query_abs_dirs()
        return self.query_hg_revision(dirs['abs_from_dir'])

    def query_to_revision(self):
        """ Shortcut to get the revision for the to repo
            """
        dirs = self.query_abs_dirs()
        return self.query_hg_revision(dirs['abs_to_dir'])

    def hg_merge_via_debugsetparents(self, cwd, old_head, new_head,
                                     preserve_tags=True, user=None):
        """ Merge 2 heads avoiding non-fastforward commits
            """
        hg = self.query_exe('hg', return_type='list')
        cmd = hg + ['debugsetparents', new_head, old_head]
        self.run_command(cmd, cwd=cwd, error_list=HgErrorList,
                         halt_on_failure=True)
        self.hg_commit(
            cwd,
            message="Merge old head via |hg debugsetparents %s %s|. "
            "CLOSED TREE DONTBUILD a=release" % (new_head, old_head),
            user=user
        )
        if preserve_tags:
            # I don't know how to do this elegantly.
            # I'm reverting .hgtags to old_head, then appending the new tags
            # from new_head to .hgtags, and hoping nothing goes wrong.
            # I'd rather not write patch files from scratch, so this seems
            # like a slightly more complex but less objectionable method?
            self.info("Trying to preserve tags from before debugsetparents...")
            dirs = self.query_abs_dirs()
            patch_file = os.path.join(dirs['abs_work_dir'], 'patch_file')
            self.run_command(
                subprocess.list2cmdline(hg + ['diff', '-r', old_head, '.hgtags', '-U9', '>', patch_file]),
                cwd=cwd,
            )
            self.run_command(
                ['patch', '-R', '-p1', '-i', patch_file],
                cwd=cwd,
                halt_on_failure=True,
            )
            tag_diff = self.read_from_file(patch_file)
            with self.opened(os.path.join(cwd, '.hgtags'), open_mode='a') as (fh, err):
                if err:
                    self.fatal("Can't append to .hgtags!")
                for n, line in enumerate(tag_diff.splitlines()):
                    # The first 4 lines of a patch are headers, so we ignore them.
                    if n < 5:
                        continue
                    # Even after that, the only lines we really care about are
                    # additions to the file.
                    # TODO: why do we only care about additions? I couldn't
                    # figure that out by reading this code.
                    if not line.startswith('+'):
                        continue
                    line = line.replace('+', '')
                    (changeset, tag) = line.split(' ')
                    if len(changeset) != 40:
                        continue
                    fh.write("%s\n" % line)
            out = self.get_output_from_command(['hg', 'status', '.hgtags'],
                                               cwd=cwd)
            if out:
                self.hg_commit(
                    cwd,
                    message="Preserve old tags after debugsetparents. "
                    "CLOSED TREE DONTBUILD a=release",
                    user=user,
                )
            else:
                self.info(".hgtags file is identical, no need to commit")

    def remove_locales(self, file_name, locales):
        """ Remove locales from shipped-locales (m-r only)
            """
        contents = self.read_from_file(file_name)
        new_contents = ""
        for line in contents.splitlines():
            locale = line.split()[0]
            if locale not in locales:
                new_contents += "%s\n" % line
            else:
                self.info("Removed locale: %s" % locale)
        self.write_to_file(file_name, new_contents)

    def touch_clobber_file(self, cwd):
        clobber_file = os.path.join(cwd, 'CLOBBER')
        contents = self.read_from_file(clobber_file)
        new_contents = ""
        for line in contents.splitlines():
            line = line.strip()
            if line.startswith("#") or line == '':
                new_contents += "%s\n" % line
        new_contents += "Merge day clobber"
        self.write_to_file(clobber_file, new_contents)

    def bump_version(self, cwd, curr_version, next_version, curr_suffix,
                     next_suffix, bump_major=False):
        """ Bump versions (m-c, m-a, m-b).

            At some point we may want to unhardcode these filenames into config
            """
        curr_weave_version = str(int(curr_version) + 2)
        next_weave_version = str(int(curr_weave_version) + 1)
        for f in self.config["version_files"]:
            from_ = "%s.0%s" % (curr_version, curr_suffix)
            to = "%s.0%s%s" % (next_version, next_suffix, f["suffix"])
            self.replace(os.path.join(cwd, f["file"]), from_, to)

        # only applicable for m-c
        if bump_major:
            self.replace(
                os.path.join(cwd, "xpcom/components/Module.h"),
                "static const unsigned int kVersion = %s;" % curr_version,
                "static const unsigned int kVersion = %s;" % next_version
            )
            self.replace(
                os.path.join(cwd, "services/sync/moz.build"),
                "DEFINES['weave_version'] = '1.%s.0'" % curr_weave_version,
                "DEFINES['weave_version'] = '1.%s.0'" % next_weave_version
            )

    # Branch-specific workflow helper methods {{{1
    def central_to_aurora(self, end_tag):
        """ mozilla-central -> mozilla-aurora behavior.

            We could have all of these individually toggled by flags, but
            by separating into workflow methods we can be more precise about
            what happens in each workflow, while allowing for things like
            staging beta user repo migrations.
            """
        dirs = self.query_abs_dirs()
        self.info("Reverting locales")
        hg = self.query_exe("hg", return_type="list")
        for f in self.config["locale_files"]:
            self.run_command(
                hg + ["revert", "-r", end_tag, f],
                cwd=dirs['abs_to_dir'],
                error_list=HgErrorList,
                halt_on_failure=True,
            )
        next_ma_version = self.get_version(dirs['abs_to_dir'])[0]
        self.bump_version(dirs['abs_to_dir'], next_ma_version, next_ma_version, "a1", "a2")
        self.apply_replacements()
        # bump m-c version
        curr_mc_version = self.get_version(dirs['abs_from_dir'])[0]
        next_mc_version = str(int(curr_mc_version) + 1)
        self.bump_version(
            dirs['abs_from_dir'], curr_mc_version, next_mc_version, "a1", "a1",
            bump_major=True
        )
        # touch clobber files
        self.touch_clobber_file(dirs['abs_from_dir'])
        self.touch_clobber_file(dirs['abs_to_dir'])

    def aurora_to_beta(self, *args, **kwargs):
        """ mozilla-aurora -> mozilla-beta behavior.

            We could have all of these individually toggled by flags, but
            by separating into workflow methods we can be more precise about
            what happens in each workflow, while allowing for things like
            staging beta user repo migrations.
            """
        dirs = self.query_abs_dirs()
        mb_version = self.get_version(dirs['abs_to_dir'])[0]
        self.bump_version(dirs['abs_to_dir'], mb_version, mb_version, "a2", "")
        self.apply_replacements()
        self.touch_clobber_file(dirs['abs_to_dir'])
        # TODO mozconfig diffing
        # The build/tools version only checks the mozconfigs from hgweb, so
        # can't help pre-push.  The in-tree mozconfig diffing requires a mach
        # virtualenv to be installed.  If we want this sooner we can put this
        # in the push action; otherwise we may just wait until we have in-tree
        # mozconfig checking.

    def beta_to_release(self, *args, **kwargs):
        """ mozilla-beta -> mozilla-release behavior.

            We could have all of these individually toggled by flags, but
            by separating into workflow methods we can be more precise about
            what happens in each workflow, while allowing for things like
            staging beta user repo migrations.
            """
        dirs = self.query_abs_dirs()
        # Reset display_version.txt
        for f in self.config["copy_files"]:
            self.copyfile(
                os.path.join(dirs['abs_to_dir'], f["src"]),
                os.path.join(dirs['abs_to_dir'], f["dst"]))

        self.apply_replacements()
        if self.config.get("remove_locales"):
            self.remove_locales(
                os.path.join(dirs['abs_to_dir'], "browser/locales/shipped-locales"),
                self.config['remove_locales']
            )
        self.touch_clobber_file(dirs['abs_to_dir'])

    def release_to_esr(self, *args, **kwargs):
        """ mozilla-release -> mozilla-esrNN behavior. """
        dirs = self.query_abs_dirs()
        for to_transplant in self.config.get("transplant_patches", []):
            self.transplant(repo=to_transplant["repo"],
                            changeset=to_transplant["changeset"],
                            cwd=dirs['abs_to_dir'])
        self.apply_replacements()
        self.touch_clobber_file(dirs['abs_to_dir'])

    def apply_replacements(self):
        dirs = self.query_abs_dirs()
        for f, from_, to in self.config["replacements"]:
            self.replace(os.path.join(dirs['abs_to_dir'], f), from_, to)

    def transplant(self, repo, changeset, cwd):
        """Transplant a Mercurial changeset from a remote repository."""
        hg = self.query_exe("hg", return_type="list")
        cmd = hg + ["--config", "extensions.transplant=", "transplant",
                    "--source", repo, changeset]
        self.info("Transplanting %s from %s" % (changeset, repo))
        status = self.run_command(
            cmd,
            cwd=cwd,
            error_list=HgErrorList,
        )
        if status != 0:
            self.fatal("Cannot transplant %s from %s properly" %
                       (changeset, repo))

    def pull_from_repo(self, from_dir, to_dir, revision=None, branch=None):
        """ Pull from one repo to another. """
        hg = self.query_exe("hg", return_type="list")
        cmd = hg + ["pull"]
        if revision:
            cmd.extend(["-r", revision])
        cmd.append(from_dir)
        self.run_command(
            cmd,
            cwd=to_dir,
            error_list=HgErrorList,
            halt_on_failure=True,
        )
        cmd = hg + ["update", "-C"]
        if branch or revision:
            cmd.extend(["-r", branch or revision])
        self.run_command(
            cmd,
            cwd=to_dir,
            error_list=HgErrorList,
            halt_on_failure=True,
        )

# Actions {{{1
    def bump_second_digit(self, *args, **kwargs):
        """Bump second digit.

         ESR need only the second digit bumped as a part of merge day."""
        dirs = self.query_abs_dirs()
        version = self.get_version(dirs['abs_to_dir'])
        curr_version = ".".join(version)
        next_version = list(version)
        # bump the second digit
        next_version[1] = str(int(next_version[1]) + 1)
        # Take major+minor and append '0' accordng to Firefox version schema.
        # 52.0 will become 52.1.0, not 52.1
        next_version = ".".join(next_version[:2] + ['0'])
        for f in self.config["version_files"]:
            self.replace(os.path.join(dirs['abs_to_dir'], f["file"]),
                         curr_version, next_version + f["suffix"])
        self.touch_clobber_file(dirs['abs_to_dir'])

    def pull(self):
        """ Pull tools first, then clone the gecko repos
            """
        repos = [{
            "repo": self.config["tools_repo_url"],
            "branch": self.config["tools_repo_branch"],
            "dest": "tools",
            "vcs": "hg",
        }] + self.query_repos()
        super(GeckoMigration, self).pull(repos=repos)

    def lock_update_paths(self):
        self.lock_balrog_rules(self.config["balrog_rules_to_lock"])

    def migrate(self):
        """ Perform the migration.
            """
        dirs = self.query_abs_dirs()
        from_fx_major_version = self.get_version(dirs['abs_from_dir'])[0]
        to_fx_major_version = self.get_version(dirs['abs_to_dir'])[0]
        base_from_rev = self.query_from_revision()
        base_to_rev = self.query_to_revision()
        base_tag = self.config['base_tag'] % {'major_version': from_fx_major_version}
        end_tag = self.config['end_tag'] % {'major_version': to_fx_major_version}
        self.hg_tag(
            dirs['abs_from_dir'], base_tag, user=self.config['hg_user'],
            revision=base_from_rev,
        )
        new_from_rev = self.query_from_revision()
        self.info("New revision %s" % new_from_rev)
        pull_revision = None
        if not self.config.get("pull_all_branches"):
            pull_revision = new_from_rev
        self.pull_from_repo(
            dirs['abs_from_dir'], dirs['abs_to_dir'],
            revision=pull_revision,
            branch="default",
        )
        if self.config.get("requires_head_merge") is not False:
            self.hg_merge_via_debugsetparents(
                dirs['abs_to_dir'], old_head=base_to_rev, new_head=new_from_rev,
                user=self.config['hg_user'],
            )
        self.hg_tag(
            dirs['abs_to_dir'], end_tag, user=self.config['hg_user'],
            revision=base_to_rev, force=True,
        )
        # Call beta_to_release etc.
        if not hasattr(self, self.config['migration_behavior']):
            self.fatal("Don't know how to proceed with migration_behavior %s !" % self.config['migration_behavior'])
        getattr(self, self.config['migration_behavior'])(end_tag=end_tag)
        self.info("Verify the diff, and apply any manual changes, such as disabling features, and --commit-changes")

    def trigger_builders(self):
        """Triggers builders that should be run directly after a merge.
        There are two different types of things we trigger:
        1) Nightly builds ("post_merge_nightly_branches" in the config).
           These are triggered with buildapi's nightly build endpoint to avoid
           duplicating all of the nightly builder names into the gecko
           migration mozharness configs. (Which would surely get out of date
           very quickly).
        2) Arbitrary builders ("post_merge_builders"). These are additional
           builders to trigger that aren't part of the nightly builder set.
           Previous example: hg bundle generation builders.
        """
        dirs = self.query_abs_dirs()
        branch = self.config["to_repo_url"].rstrip("/").split("/")[-1]
        revision = self.query_to_revision()
        # Horrible hack because our internal buildapi interface doesn't let us
        # actually do anything. Need to use the public one w/ auth.
        username = raw_input("LDAP Username: ")
        password = getpass(prompt="LDAP Password: ")
        auth = (username, password)
        for builder in self.config["post_merge_builders"]:
            self.trigger_arbitrary_job(builder, branch, revision, auth)
        for nightly_branch in self.config["post_merge_nightly_branches"]:
            nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch))
            self.trigger_nightly_builds(nightly_branch, nightly_revision, auth)

# __main__ {{{1
if __name__ == '__main__':
    GeckoMigration().run_and_exit()