diff options
Diffstat (limited to 'testing/mozharness/scripts/mobile_l10n.py')
| -rwxr-xr-x | testing/mozharness/scripts/mobile_l10n.py | 714 |
1 files changed, 714 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/mobile_l10n.py b/testing/mozharness/scripts/mobile_l10n.py new file mode 100755 index 000000000..cbac6fa67 --- /dev/null +++ b/testing/mozharness/scripts/mobile_l10n.py @@ -0,0 +1,714 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""mobile_l10n.py + +This currently supports nightly and release single locale repacks for +Android. This also creates nightly updates. +""" + +from copy import deepcopy +import os +import re +import subprocess +import sys +import time +import shlex + +try: + import simplejson as json + assert json +except ImportError: + import json + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(sys.path[0])) + +from mozharness.base.errors import BaseErrorList, MakefileErrorList +from mozharness.base.log import OutputParser +from mozharness.base.transfer import TransferMixin +from mozharness.mozilla.buildbot import BuildbotMixin +from mozharness.mozilla.purge import PurgeMixin +from mozharness.mozilla.release import ReleaseMixin +from mozharness.mozilla.signing import MobileSigningMixin +from mozharness.mozilla.tooltool import TooltoolMixin +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.l10n.locales import LocalesMixin +from mozharness.mozilla.mock import MockMixin +from mozharness.mozilla.updates.balrog import BalrogMixin +from mozharness.base.python import VirtualenvMixin +from mozharness.mozilla.taskcluster_helper import Taskcluster + + +# MobileSingleLocale {{{1 +class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, + MobileSigningMixin, TransferMixin, TooltoolMixin, + BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin, + VirtualenvMixin): + config_options = [[ + ['--locale', ], + {"action": "extend", + "dest": "locales", + "type": "string", + "help": "Specify the locale(s) to sign and update" + } + ], [ + ['--locales-file', ], + {"action": "store", + "dest": "locales_file", + "type": "string", + "help": "Specify a file to determine which locales to sign and update" + } + ], [ + ['--tag-override', ], + {"action": "store", + "dest": "tag_override", + "type": "string", + "help": "Override the tags set for all repos" + } + ], [ + ['--user-repo-override', ], + {"action": "store", + "dest": "user_repo_override", + "type": "string", + "help": "Override the user repo path for all repos" + } + ], [ + ['--release-config-file', ], + {"action": "store", + "dest": "release_config_file", + "type": "string", + "help": "Specify the release config file to use" + } + ], [ + ['--key-alias', ], + {"action": "store", + "dest": "key_alias", + "type": "choice", + "default": "nightly", + "choices": ["nightly", "release"], + "help": "Specify the signing key alias" + } + ], [ + ['--this-chunk', ], + {"action": "store", + "dest": "this_locale_chunk", + "type": "int", + "help": "Specify which chunk of locales to run" + } + ], [ + ['--total-chunks', ], + {"action": "store", + "dest": "total_locale_chunks", + "type": "int", + "help": "Specify the total number of chunks of locales" + } + ], [ + ["--disable-mock"], + {"dest": "disable_mock", + "action": "store_true", + "help": "do not run under mock despite what gecko-config says", + } + ], [ + ['--revision', ], + {"action": "store", + "dest": "revision", + "type": "string", + "help": "Override the gecko revision to use (otherwise use buildbot supplied" + " value, or en-US revision) "} + ]] + + def __init__(self, require_config_file=True): + buildscript_kwargs = { + 'all_actions': [ + "clobber", + "pull", + "clone-locales", + "list-locales", + "setup", + "repack", + "validate-repacks-signed", + "upload-repacks", + "create-virtualenv", + "taskcluster-upload", + "submit-to-balrog", + "summary", + ], + 'config': { + 'taskcluster_credentials_file': 'oauth.txt', + 'virtualenv_modules': [ + 'requests==2.8.1', + 'PyHawk-with-a-single-extra-commit==0.1.5', + 'taskcluster==0.0.26', + ], + 'virtualenv_path': 'venv', + }, + } + LocalesMixin.__init__(self) + MercurialScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + **buildscript_kwargs + ) + self.base_package_name = None + self.buildid = None + self.make_ident_output = None + self.repack_env = None + self.revision = None + self.upload_env = None + self.version = None + self.upload_urls = {} + self.locales_property = {} + + # Helper methods {{{2 + def query_repack_env(self): + if self.repack_env: + return self.repack_env + c = self.config + replace_dict = {} + if c.get('release_config_file'): + rc = self.query_release_config() + replace_dict = { + 'version': rc['version'], + 'buildnum': rc['buildnum'] + } + repack_env = self.query_env(partial_env=c.get("repack_env"), + replace_dict=replace_dict) + if c.get('base_en_us_binary_url') and c.get('release_config_file'): + rc = self.query_release_config() + repack_env['EN_US_BINARY_URL'] = c['base_en_us_binary_url'] % replace_dict + if 'MOZ_SIGNING_SERVERS' in os.environ: + repack_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd(formats=['jar'])) + self.repack_env = repack_env + return self.repack_env + + def query_l10n_env(self): + return self.query_env() + + def query_upload_env(self): + if self.upload_env: + return self.upload_env + c = self.config + replace_dict = { + 'buildid': self.query_buildid(), + 'version': self.query_version(), + } + replace_dict.update(c) + + # Android l10n builds use a non-standard location for l10n files. Other + # builds go to 'mozilla-central-l10n', while android builds add part of + # the platform name as well, like 'mozilla-central-android-api-15-l10n'. + # So we override the branch with something that contains the platform + # name. + replace_dict['branch'] = c['upload_branch'] + replace_dict['post_upload_extra'] = ' '.join(c.get('post_upload_extra', [])) + + upload_env = self.query_env(partial_env=c.get("upload_env"), + replace_dict=replace_dict) + if 'MOZ_SIGNING_SERVERS' in os.environ: + upload_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd()) + if self.query_is_release_or_beta(): + upload_env['MOZ_PKG_VERSION'] = '%(version)s' % replace_dict + self.upload_env = upload_env + return self.upload_env + + def _query_make_ident_output(self): + """Get |make ident| output from the objdir. + Only valid after setup is run. + """ + if self.make_ident_output: + return self.make_ident_output + env = self.query_repack_env() + dirs = self.query_abs_dirs() + output = self.get_output_from_command_m(["make", "ident"], + cwd=dirs['abs_locales_dir'], + env=env, + silent=True, + halt_on_failure=True) + parser = OutputParser(config=self.config, log_obj=self.log_obj, + error_list=MakefileErrorList) + parser.add_lines(output) + self.make_ident_output = output + return output + + def query_buildid(self): + """Get buildid from the objdir. + Only valid after setup is run. + """ + if self.buildid: + return self.buildid + r = re.compile("buildid (\d+)") + output = self._query_make_ident_output() + for line in output.splitlines(): + m = r.match(line) + if m: + self.buildid = m.groups()[0] + return self.buildid + + def query_revision(self): + """Get revision from the objdir. + Only valid after setup is run. + """ + if self.revision: + return self.revision + r = re.compile(r"gecko_revision ([0-9a-f]+\+?)") + output = self._query_make_ident_output() + for line in output.splitlines(): + m = r.match(line) + if m: + self.revision = m.groups()[0] + return self.revision + + def _query_make_variable(self, variable, make_args=None): + make = self.query_exe('make') + env = self.query_repack_env() + dirs = self.query_abs_dirs() + if make_args is None: + make_args = [] + # TODO error checking + output = self.get_output_from_command_m( + [make, "echo-variable-%s" % variable] + make_args, + cwd=dirs['abs_locales_dir'], silent=True, + env=env + ) + parser = OutputParser(config=self.config, log_obj=self.log_obj, + error_list=MakefileErrorList) + parser.add_lines(output) + return output.strip() + + def query_base_package_name(self): + """Get the package name from the objdir. + Only valid after setup is run. + """ + if self.base_package_name: + return self.base_package_name + self.base_package_name = self._query_make_variable( + "PACKAGE", + make_args=['AB_CD=%(locale)s'] + ) + return self.base_package_name + + def query_version(self): + """Get the package name from the objdir. + Only valid after setup is run. + """ + if self.version: + return self.version + c = self.config + if c.get('release_config_file'): + rc = self.query_release_config() + self.version = rc['version'] + else: + self.version = self._query_make_variable("MOZ_APP_VERSION") + return self.version + + def query_upload_url(self, locale): + if locale in self.upload_urls: + return self.upload_urls[locale] + else: + self.error("Can't determine the upload url for %s!" % locale) + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(MobileSingleLocale, self).query_abs_dirs() + + dirs = { + 'abs_tools_dir': + os.path.join(abs_dirs['base_work_dir'], 'tools'), + 'build_dir': + os.path.join(abs_dirs['base_work_dir'], 'build'), + } + + abs_dirs.update(dirs) + self.abs_dirs = abs_dirs + return self.abs_dirs + + def add_failure(self, locale, message, **kwargs): + self.locales_property[locale] = "Failed" + prop_key = "%s_failure" % locale + prop_value = self.query_buildbot_property(prop_key) + if prop_value: + prop_value = "%s %s" % (prop_value, message) + else: + prop_value = message + self.set_buildbot_property(prop_key, prop_value, write_to_file=True) + MercurialScript.add_failure(self, locale, message=message, **kwargs) + + def summary(self): + MercurialScript.summary(self) + # TODO we probably want to make this configurable on/off + locales = self.query_locales() + for locale in locales: + self.locales_property.setdefault(locale, "Success") + self.set_buildbot_property("locales", json.dumps(self.locales_property), write_to_file=True) + + # Actions {{{2 + def clobber(self): + self.read_buildbot_config() + dirs = self.query_abs_dirs() + c = self.config + objdir = os.path.join(dirs['abs_work_dir'], c['mozilla_dir'], + c['objdir']) + super(MobileSingleLocale, self).clobber(always_clobber_dirs=[objdir]) + + def pull(self): + c = self.config + dirs = self.query_abs_dirs() + repos = [] + replace_dict = {} + if c.get("user_repo_override"): + replace_dict['user_repo_override'] = c['user_repo_override'] + # deepcopy() needed because of self.config lock bug :( + for repo_dict in deepcopy(c['repos']): + repo_dict['repo'] = repo_dict['repo'] % replace_dict + repos.append(repo_dict) + else: + repos = c['repos'] + self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'], + tag_override=c.get('tag_override')) + + def clone_locales(self): + self.pull_locale_source() + + # list_locales() is defined in LocalesMixin. + + def _setup_configure(self, buildid=None): + c = self.config + dirs = self.query_abs_dirs() + env = self.query_repack_env() + make = self.query_exe("make") + if self.run_command_m([make, "-f", "client.mk", "configure"], + cwd=dirs['abs_mozilla_dir'], + env=env, + error_list=MakefileErrorList): + self.fatal("Configure failed!") + + # Run 'make export' in objdir/config to get nsinstall + self.run_command_m([make, 'export'], + cwd=os.path.join(dirs['abs_objdir'], 'config'), + env=env, + error_list=MakefileErrorList, + halt_on_failure=True) + + # Run 'make buildid.h' in objdir/ to get the buildid.h file + cmd = [make, 'buildid.h'] + if buildid: + cmd.append('MOZ_BUILD_DATE=%s' % str(buildid)) + self.run_command_m(cmd, + cwd=dirs['abs_objdir'], + env=env, + error_list=MakefileErrorList, + halt_on_failure=True) + + def setup(self): + c = self.config + dirs = self.query_abs_dirs() + mozconfig_path = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig') + self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']), + mozconfig_path) + # TODO stop using cat + cat = self.query_exe("cat") + make = self.query_exe("make") + self.run_command_m([cat, mozconfig_path]) + env = self.query_repack_env() + if self.config.get("tooltool_config"): + self.tooltool_fetch( + self.config['tooltool_config']['manifest'], + output_dir=self.config['tooltool_config']['output_dir'] % self.query_abs_dirs(), + ) + self._setup_configure() + self.run_command_m([make, "wget-en-US"], + cwd=dirs['abs_locales_dir'], + env=env, + error_list=MakefileErrorList, + halt_on_failure=True) + self.run_command_m([make, "unpack"], + cwd=dirs['abs_locales_dir'], + env=env, + error_list=MakefileErrorList, + halt_on_failure=True) + + # on try we want the source we already have, otherwise update to the + # same as the en-US binary + if self.config.get("update_gecko_source_to_enUS", True): + revision = self.query_revision() + if not revision: + self.fatal("Can't determine revision!") + hg = self.query_exe("hg") + # TODO do this through VCSMixin instead of hardcoding hg + self.run_command_m([hg, "update", "-r", revision], + cwd=dirs["abs_mozilla_dir"], + env=env, + error_list=BaseErrorList, + halt_on_failure=True) + self.set_buildbot_property('revision', revision, write_to_file=True) + # Configure again since the hg update may have invalidated it. + buildid = self.query_buildid() + self._setup_configure(buildid=buildid) + + def repack(self): + # TODO per-locale logs and reporting. + dirs = self.query_abs_dirs() + locales = self.query_locales() + make = self.query_exe("make") + repack_env = self.query_repack_env() + success_count = total_count = 0 + for locale in locales: + total_count += 1 + self.enable_mock() + result = self.run_compare_locales(locale) + self.disable_mock() + if result: + self.add_failure(locale, message="%s failed in compare-locales!" % locale) + continue + if self.run_command_m([make, "installers-%s" % locale], + cwd=dirs['abs_locales_dir'], + env=repack_env, + error_list=MakefileErrorList, + halt_on_failure=False): + self.add_failure(locale, message="%s failed in make installers-%s!" % (locale, locale)) + continue + success_count += 1 + self.summarize_success_count(success_count, total_count, + message="Repacked %d of %d binaries successfully.") + + def validate_repacks_signed(self): + c = self.config + dirs = self.query_abs_dirs() + locales = self.query_locales() + base_package_name = self.query_base_package_name() + base_package_dir = os.path.join(dirs['abs_objdir'], 'dist') + repack_env = self.query_repack_env() + success_count = total_count = 0 + for locale in locales: + total_count += 1 + signed_path = os.path.join(base_package_dir, + base_package_name % {'locale': locale}) + # We need to wrap what this function does with mock, since + # MobileSigningMixin doesn't know about mock + self.enable_mock() + status = self.verify_android_signature( + signed_path, + script=c['signature_verification_script'], + env=repack_env, + key_alias=c['key_alias'], + ) + self.disable_mock() + if status: + self.add_failure(locale, message="Errors verifying %s binary!" % locale) + # No need to rm because upload is per-locale + continue + success_count += 1 + self.summarize_success_count(success_count, total_count, + message="Validated signatures on %d of %d binaries successfully.") + + def taskcluster_upload(self): + auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) + credentials = {} + execfile(auth, credentials) + client_id = credentials.get('taskcluster_clientId') + access_token = credentials.get('taskcluster_accessToken') + if not client_id or not access_token: + self.warning('Skipping S3 file upload: No taskcluster credentials.') + return + + self.activate_virtualenv() + + dirs = self.query_abs_dirs() + locales = self.query_locales() + make = self.query_exe("make") + upload_env = self.query_upload_env() + cwd = dirs['abs_locales_dir'] + branch = self.config['branch'] + revision = self.query_revision() + repo = self.query_l10n_repo() + pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hg') + pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) + routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], + 'testing/mozharness/configs/routes.json') + with open(routes_json) as routes_file: + contents = json.load(routes_file) + templates = contents['l10n'] + + for locale in locales: + output = self.get_output_from_command_m( + "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale), + cwd=cwd, + env=upload_env, + ) + files = shlex.split(output) + abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files] + + routes = [] + fmt = { + 'index': self.config.get('taskcluster_index', 'index.garbage.staging'), + 'project': branch, + 'head_rev': revision, + 'pushdate': pushdate, + 'year': pushdate[0:4], + 'month': pushdate[4:6], + 'day': pushdate[6:8], + 'build_product': self.config['stage_product'], + 'build_name': self.query_build_name(), + 'build_type': self.query_build_type(), + 'locale': locale, + } + for template in templates: + routes.append(template.format(**fmt)) + + self.info('Using routes: %s' % routes) + tc = Taskcluster(branch, + pushinfo.pushdate, # Use pushdate as the rank + client_id, + access_token, + self.log_obj, + ) + task = tc.create_task(routes) + tc.claim_task(task) + + for upload_file in abs_files: + tc.create_artifact(task, upload_file) + tc.report_completed(task) + + def upload_repacks(self): + c = self.config + dirs = self.query_abs_dirs() + locales = self.query_locales() + make = self.query_exe("make") + base_package_name = self.query_base_package_name() + version = self.query_version() + upload_env = self.query_upload_env() + success_count = total_count = 0 + buildnum = None + if c.get('release_config_file'): + rc = self.query_release_config() + buildnum = rc['buildnum'] + for locale in locales: + if self.query_failure(locale): + self.warning("Skipping previously failed locale %s." % locale) + continue + total_count += 1 + if c.get('base_post_upload_cmd'): + upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))} + output = self.get_output_from_command_m( + # Ugly hack to avoid |make upload| stderr from showing up + # as get_output_from_command errors + "%s upload AB_CD=%s 2>&1" % (make, locale), + cwd=dirs['abs_locales_dir'], + env=upload_env, + silent=True + ) + parser = OutputParser(config=self.config, log_obj=self.log_obj, + error_list=MakefileErrorList) + parser.add_lines(output) + if parser.num_errors: + self.add_failure(locale, message="%s failed in make upload!" % (locale)) + continue + package_name = base_package_name % {'locale': locale} + r = re.compile("(http.*%s)" % package_name) + for line in output.splitlines(): + m = r.match(line) + if m: + self.upload_urls[locale] = m.groups()[0] + self.info("Found upload url %s" % self.upload_urls[locale]) + success_count += 1 + self.summarize_success_count(success_count, total_count, + message="Make Upload for %d of %d locales successful.") + + def checkout_tools(self): + dirs = self.query_abs_dirs() + + # We need hg.m.o/build/tools checked out + self.info("Checking out tools") + repos = [{ + 'repo': self.config['tools_repo'], + 'vcs': "hg", + 'branch': "default", + 'dest': dirs['abs_tools_dir'], + }] + rev = self.vcs_checkout(**repos[0]) + self.set_buildbot_property("tools_revision", rev, write_to_file=True) + + def query_apkfile_path(self,locale): + + dirs = self.query_abs_dirs() + apkdir = os.path.join(dirs['abs_objdir'], 'dist') + r = r"(\.)" + re.escape(locale) + r"(\.*)" + + apks = [] + for f in os.listdir(apkdir): + if f.endswith(".apk") and re.search(r, f): + apks.append(f) + if len(apks) == 0: + self.fatal("Found no apks files in %s, don't know what to do:\n%s" % (apkdir, apks), exit_code=1) + + return os.path.join(apkdir, apks[0]) + + def query_is_release_or_beta(self): + + return bool(self.config.get("is_release_or_beta")) + + def submit_to_balrog(self): + + if not self.query_is_nightly() and not self.query_is_release_or_beta(): + self.info("Not a nightly or release build, skipping balrog submission.") + return + + if not self.config.get("balrog_servers"): + self.info("balrog_servers not set; skipping balrog submission.") + return + + self.checkout_tools() + + dirs = self.query_abs_dirs() + locales = self.query_locales() + balrogReady = True + for locale in locales: + apk_url = self.query_upload_url(locale) + if not apk_url: + self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale)) + balrogReady = False + continue + if not balrogReady: + return self.fatal(message="Not all repacks successful, abort without submitting to balrog") + + for locale in locales: + apkfile = self.query_apkfile_path(locale) + apk_url = self.query_upload_url(locale) + + # Set other necessary properties for Balrog submission. None need to + # be passed back to buildbot, so we won't write them to the properties + #files. + self.set_buildbot_property("locale", locale) + + self.set_buildbot_property("appVersion", self.query_version()) + # The Balrog submitter translates this platform into a build target + # via https://github.com/mozilla/build-tools/blob/master/lib/python/release/platforms.py#L23 + self.set_buildbot_property("platform", self.buildbot_config["properties"]["platform"]) + #TODO: Is there a better way to get this? + + self.set_buildbot_property("appName", "Fennec") + # TODO: don't hardcode + self.set_buildbot_property("hashType", "sha512") + self.set_buildbot_property("completeMarSize", self.query_filesize(apkfile)) + self.set_buildbot_property("completeMarHash", self.query_sha512sum(apkfile)) + self.set_buildbot_property("completeMarUrl", apk_url) + self.set_buildbot_property("isOSUpdate", False) + self.set_buildbot_property("buildid", self.query_buildid()) + + if self.query_is_nightly(): + self.submit_balrog_updates(release_type="nightly") + else: + self.submit_balrog_updates(release_type="release") + if not self.query_is_nightly(): + self.submit_balrog_release_pusher(dirs) + +# main {{{1 +if __name__ == '__main__': + single_locale = MobileSingleLocale() + single_locale.run_and_exit() |