Add m-esr52 at 52.6.0

1 files changed, 1152 insertions, 0 deletions

diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py
new file mode 100755
index 000000000..0626ce35b
--- /dev/null
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -0,0 +1,1152 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""desktop_l10n.py
+
+This script manages Desktop repacks for nightly builds.
+"""
+import os
+import re
+import sys
+import time
+import shlex
+import subprocess
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import BaseErrorList, MakefileErrorList
+from mozharness.base.script import BaseScript
+from mozharness.base.transfer import TransferMixin
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.purge import PurgeMixin
+from mozharness.mozilla.building.buildbase import MakeUploadOutputParser
+from mozharness.mozilla.l10n.locales import LocalesMixin
+from mozharness.mozilla.mar import MarMixin
+from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.release import ReleaseMixin
+from mozharness.mozilla.signing import SigningMixin
+from mozharness.mozilla.updates.balrog import BalrogMixin
+from mozharness.mozilla.taskcluster_helper import Taskcluster
+from mozharness.base.python import VirtualenvMixin
+from mozharness.mozilla.mock import ERROR_MSGS
+
+try:
+    import simplejson as json
+    assert json
+except ImportError:
+    import json
+
+
+# needed by _map
+SUCCESS = 0
+FAILURE = 1
+
+SUCCESS_STR = "Success"
+FAILURE_STR = "Failed"
+
+# when running get_output_form_command, pymake has some extra output
+# that needs to be filtered out
+PyMakeIgnoreList = [
+    re.compile(r'''.*make\.py(?:\[\d+\])?: Entering directory'''),
+    re.compile(r'''.*make\.py(?:\[\d+\])?: Leaving directory'''),
+]
+
+
+# mandatory configuration options, without them, this script will not work
+# it's a list of values that are already known before starting a build
+configuration_tokens = ('branch',
+                        'platform',
+                        'update_platform',
+                        'update_channel',
+                        'ssh_key_dir',
+                        'stage_product',
+                        'upload_environment',
+                       )
+# some other values such as "%(version)s", "%(buildid)s", ...
+# are defined at run time and they cannot be enforced in the _pre_config_lock
+# phase
+runtime_config_tokens = ('buildid', 'version', 'locale', 'from_buildid',
+                         'abs_objdir', 'abs_merge_dir', 'revision',
+                         'to_buildid', 'en_us_binary_url', 'mar_tools_url',
+                         'post_upload_extra', 'who')
+
+# DesktopSingleLocale {{{1
+class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
+                          VCSMixin, SigningMixin, PurgeMixin, BaseScript,
+                          BalrogMixin, MarMixin, VirtualenvMixin, TransferMixin):
+    """Manages desktop repacks"""
+    config_options = [[
+        ['--balrog-config', ],
+        {"action": "extend",
+         "dest": "config_files",
+         "type": "string",
+         "help": "Specify the balrog configuration file"}
+    ], [
+        ['--branch-config', ],
+        {"action": "extend",
+         "dest": "config_files",
+         "type": "string",
+         "help": "Specify the branch configuration file"}
+    ], [
+        ['--environment-config', ],
+        {"action": "extend",
+         "dest": "config_files",
+         "type": "string",
+         "help": "Specify the environment (staging, production, ...) configuration file"}
+    ], [
+        ['--platform-config', ],
+        {"action": "extend",
+         "dest": "config_files",
+         "type": "string",
+         "help": "Specify the platform configuration file"}
+    ], [
+        ['--locale', ],
+        {"action": "extend",
+         "dest": "locales",
+         "type": "string",
+         "help": "Specify the locale(s) to sign and update. Optionally pass"
+                 " revision separated by colon, en-GB:default."}
+    ], [
+        ['--locales-file', ],
+        {"action": "store",
+         "dest": "locales_file",
+         "type": "string",
+         "help": "Specify a file to determine which locales to sign and update"}
+    ], [
+        ['--tag-override', ],
+        {"action": "store",
+         "dest": "tag_override",
+         "type": "string",
+         "help": "Override the tags set for all repos"}
+    ], [
+        ['--revision', ],
+        {"action": "store",
+         "dest": "revision",
+         "type": "string",
+         "help": "Override the gecko revision to use (otherwise use buildbot supplied"
+                 " value, or en-US revision) "}
+    ], [
+        ['--user-repo-override', ],
+        {"action": "store",
+         "dest": "user_repo_override",
+         "type": "string",
+         "help": "Override the user repo path for all repos"}
+    ], [
+        ['--release-config-file', ],
+        {"action": "store",
+         "dest": "release_config_file",
+         "type": "string",
+         "help": "Specify the release config file to use"}
+    ], [
+        ['--this-chunk', ],
+        {"action": "store",
+         "dest": "this_locale_chunk",
+         "type": "int",
+         "help": "Specify which chunk of locales to run"}
+    ], [
+        ['--total-chunks', ],
+        {"action": "store",
+         "dest": "total_locale_chunks",
+         "type": "int",
+         "help": "Specify the total number of chunks of locales"}
+    ], [
+        ['--en-us-installer-url', ],
+        {"action": "store",
+         "dest": "en_us_installer_url",
+         "type": "string",
+         "help": "Specify the url of the en-us binary"}
+    ], [
+        ["--disable-mock"], {
+        "dest": "disable_mock",
+        "action": "store_true",
+        "help": "do not run under mock despite what gecko-config says"}
+    ]]
+
+    def __init__(self, require_config_file=True):
+        # fxbuild style:
+        buildscript_kwargs = {
+            'all_actions': [
+                "clobber",
+                "pull",
+                "clone-locales",
+                "list-locales",
+                "setup",
+                "repack",
+                "taskcluster-upload",
+                "funsize-props",
+                "submit-to-balrog",
+                "summary",
+            ],
+            'config': {
+                "buildbot_json_path": "buildprops.json",
+                "ignore_locales": ["en-US"],
+                "locales_dir": "browser/locales",
+                "update_mar_dir": "dist/update",
+                "buildid_section": "App",
+                "buildid_option": "BuildID",
+                "application_ini": "application.ini",
+                "log_name": "single_locale",
+                "clobber_file": 'CLOBBER',
+                "appName": "Firefox",
+                "hashType": "sha512",
+                "taskcluster_credentials_file": "oauth.txt",
+                'virtualenv_modules': [
+                    'requests==2.8.1',
+                    'PyHawk-with-a-single-extra-commit==0.1.5',
+                    'taskcluster==0.0.26',
+                ],
+                'virtualenv_path': 'venv',
+            },
+        }
+        #
+
+        LocalesMixin.__init__(self)
+        BaseScript.__init__(
+            self,
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            **buildscript_kwargs
+        )
+
+        self.buildid = None
+        self.make_ident_output = None
+        self.bootstrap_env = None
+        self.upload_env = None
+        self.revision = None
+        self.enUS_revision = None
+        self.version = None
+        self.upload_urls = {}
+        self.locales_property = {}
+        self.package_urls = {}
+        self.pushdate = None
+        # upload_files is a dictionary of files to upload, keyed by locale.
+        self.upload_files = {}
+
+        if 'mock_target' in self.config:
+            self.enable_mock()
+
+    def _pre_config_lock(self, rw_config):
+        """replaces 'configuration_tokens' with their values, before the
+           configuration gets locked. If some of the configuration_tokens
+           are not present, stops the execution of the script"""
+        # since values as branch, platform are mandatory, can replace them in
+        # in the configuration before it is locked down
+        # mandatory tokens
+        for token in configuration_tokens:
+            if token not in self.config:
+                self.fatal('No %s in configuration!' % token)
+
+        # all the important tokens are present in our configuration
+        for token in configuration_tokens:
+            # token_string '%(branch)s'
+            token_string = ''.join(('%(', token, ')s'))
+            # token_value => ash
+            token_value = self.config[token]
+            for element in self.config:
+                # old_value =>  https://hg.mozilla.org/projects/%(branch)s
+                old_value = self.config[element]
+                # new_value => https://hg.mozilla.org/projects/ash
+                new_value = self.__detokenise_element(self.config[element],
+                                                      token_string,
+                                                      token_value)
+                if new_value and new_value != old_value:
+                    msg = "%s: replacing %s with %s" % (element,
+                                                        old_value,
+                                                        new_value)
+                    self.debug(msg)
+                    self.config[element] = new_value
+
+        # now, only runtime_config_tokens should be present in config
+        # we should parse self.config and fail if any other  we spot any
+        # other token
+        tokens_left = set(self._get_configuration_tokens(self.config))
+        unknown_tokens = set(tokens_left) - set(runtime_config_tokens)
+        if unknown_tokens:
+            msg = ['unknown tokens in configuration:']
+            for t in unknown_tokens:
+                msg.append(t)
+            self.fatal(' '.join(msg))
+        self.info('configuration looks ok')
+
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        props = self.buildbot_config["properties"]
+        for prop in ['mar_tools_url']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+
+    def _get_configuration_tokens(self, iterable):
+        """gets a list of tokens in iterable"""
+        regex = re.compile('%\(\w+\)s')
+        results = []
+        try:
+            for element in iterable:
+                if isinstance(iterable, str):
+                    # this is a string, look for tokens
+                    # self.debug("{0}".format(re.findall(regex, element)))
+                    tokens = re.findall(regex, iterable)
+                    for token in tokens:
+                        # clean %(branch)s => branch
+                        # remove %(
+                        token_name = token.partition('%(')[2]
+                        # remove )s
+                        token_name = token_name.partition(')s')[0]
+                        results.append(token_name)
+                    break
+
+                elif isinstance(iterable, (list, tuple)):
+                    results.extend(self._get_configuration_tokens(element))
+
+                elif isinstance(iterable, dict):
+                    results.extend(self._get_configuration_tokens(iterable[element]))
+
+        except TypeError:
+            # element is a int/float/..., nothing to do here
+            pass
+
+        # remove duplicates, and return results
+
+        return list(set(results))
+
+    def __detokenise_element(self, config_option, token, value):
+        """reads config_options and returns a version of the same config_option
+           replacing token with value recursively"""
+        # config_option is a string, let's replace token with value
+        if isinstance(config_option, str):
+            # if token does not appear in this string,
+            # nothing happens and the original value is returned
+            return config_option.replace(token, value)
+        # it's a dictionary
+        elif isinstance(config_option, dict):
+            # replace token for each element of this dictionary
+            for element in config_option:
+                config_option[element] = self.__detokenise_element(
+                    config_option[element], token, value)
+            return config_option
+        # it's a list
+        elif isinstance(config_option, list):
+            # create a new list and append the replaced elements
+            new_list = []
+            for element in config_option:
+                new_list.append(self.__detokenise_element(element, token, value))
+            return new_list
+        elif isinstance(config_option, tuple):
+            # create a new list and append the replaced elements
+            new_list = []
+            for element in config_option:
+                new_list.append(self.__detokenise_element(element, token, value))
+            return tuple(new_list)
+        else:
+            # everything else, bool, number, ...
+            return config_option
+
+    # Helper methods {{{2
+    def query_bootstrap_env(self):
+        """returns the env for repacks"""
+        if self.bootstrap_env:
+            return self.bootstrap_env
+        config = self.config
+        replace_dict = self.query_abs_dirs()
+
+        replace_dict['en_us_binary_url'] = config.get('en_us_binary_url')
+        self.read_buildbot_config()
+        # Override en_us_binary_url if packageUrl is passed as a property from
+        # the en-US build
+        if self.buildbot_config["properties"].get("packageUrl"):
+            packageUrl = self.buildbot_config["properties"]["packageUrl"]
+            # trim off the filename, the build system wants a directory
+            packageUrl = packageUrl.rsplit('/', 1)[0]
+            self.info("Overriding en_us_binary_url with %s" % packageUrl)
+            replace_dict['en_us_binary_url'] = str(packageUrl)
+        # Override en_us_binary_url if passed as a buildbot property
+        if self.buildbot_config["properties"].get("en_us_binary_url"):
+            self.info("Overriding en_us_binary_url with %s" %
+                      self.buildbot_config["properties"]["en_us_binary_url"])
+            replace_dict['en_us_binary_url'] = \
+                str(self.buildbot_config["properties"]["en_us_binary_url"])
+        bootstrap_env = self.query_env(partial_env=config.get("bootstrap_env"),
+                                       replace_dict=replace_dict)
+        if 'MOZ_SIGNING_SERVERS' in os.environ:
+            sign_cmd = self.query_moz_sign_cmd(formats=None)
+            sign_cmd = subprocess.list2cmdline(sign_cmd)
+            # windows fix
+            bootstrap_env['MOZ_SIGN_CMD'] = sign_cmd.replace('\\', '\\\\\\\\')
+        for binary in self._mar_binaries():
+            # "mar -> MAR" and 'mar.exe -> MAR' (windows)
+            name = binary.replace('.exe', '')
+            name = name.upper()
+            binary_path = os.path.join(self._mar_tool_dir(), binary)
+            # windows fix...
+            if binary.endswith('.exe'):
+                binary_path = binary_path.replace('\\', '\\\\\\\\')
+            bootstrap_env[name] = binary_path
+            if 'LOCALE_MERGEDIR' in bootstrap_env:
+                # windows fix
+                bootstrap_env['LOCALE_MERGEDIR'] = bootstrap_env['LOCALE_MERGEDIR'].replace('\\', '\\\\\\\\')
+        if self.query_is_nightly():
+            bootstrap_env["IS_NIGHTLY"] = "yes"
+        self.bootstrap_env = bootstrap_env
+        return self.bootstrap_env
+
+    def _query_upload_env(self):
+        """returns the environment used for the upload step"""
+        if self.upload_env:
+            return self.upload_env
+        config = self.config
+
+        replace_dict = {
+            'buildid': self._query_buildid(),
+            'version': self.query_version(),
+            'post_upload_extra': ' '.join(config.get('post_upload_extra', [])),
+            'upload_environment': config['upload_environment'],
+        }
+        if config['branch'] == 'try':
+            replace_dict.update({
+                'who': self.query_who(),
+                'revision': self._query_revision(),
+            })
+        upload_env = self.query_env(partial_env=config.get("upload_env"),
+                                    replace_dict=replace_dict)
+        # check if there are any extra option from the platform configuration
+        # and append them to the env
+
+        if 'upload_env_extra' in config:
+            for extra in config['upload_env_extra']:
+                upload_env[extra] = config['upload_env_extra'][extra]
+
+        self.upload_env = upload_env
+        return self.upload_env
+
+    def query_l10n_env(self):
+        l10n_env = self._query_upload_env().copy()
+        # both upload_env and bootstrap_env define MOZ_SIGN_CMD
+        # the one from upload_env is taken from os.environ, the one from
+        # bootstrap_env is set with query_moz_sign_cmd()
+        # we need to use the value provided my query_moz_sign_cmd or make upload
+        # will fail (signtool.py path is wrong)
+        l10n_env.update(self.query_bootstrap_env())
+        return l10n_env
+
+    def _query_make_ident_output(self):
+        """Get |make ident| output from the objdir.
+        Only valid after setup is run.
+       """
+        if self.make_ident_output:
+            return self.make_ident_output
+        dirs = self.query_abs_dirs()
+        self.make_ident_output = self._get_output_from_make(
+            target=["ident"],
+            cwd=dirs['abs_locales_dir'],
+            env=self.query_bootstrap_env())
+        return self.make_ident_output
+
+    def _query_buildid(self):
+        """Get buildid from the objdir.
+        Only valid after setup is run.
+       """
+        if self.buildid:
+            return self.buildid
+        r = re.compile(r"buildid (\d+)")
+        output = self._query_make_ident_output()
+        for line in output.splitlines():
+            match = r.match(line)
+            if match:
+                self.buildid = match.groups()[0]
+        return self.buildid
+
+    def _query_revision(self):
+        """ Get the gecko revision in this order of precedence
+              * cached value
+              * command line arg --revision   (development, taskcluster)
+              * buildbot properties           (try with buildbot forced build)
+              * buildbot change               (try with buildbot scheduler)
+              * from the en-US build          (m-c & m-a)
+
+        This will fail the last case if the build hasn't been pulled yet.
+        """
+        if self.revision:
+            return self.revision
+
+        self.read_buildbot_config()
+        config = self.config
+        revision = None
+        if config.get("revision"):
+            revision = config["revision"]
+        elif 'revision' in self.buildbot_properties:
+            revision = self.buildbot_properties['revision']
+        elif (self.buildbot_config and
+                  self.buildbot_config.get('sourcestamp', {}).get('revision')):
+            revision = self.buildbot_config['sourcestamp']['revision']
+        elif self.buildbot_config and self.buildbot_config.get('revision'):
+            revision = self.buildbot_config['revision']
+        elif config.get("update_gecko_source_to_enUS", True):
+            revision = self._query_enUS_revision()
+
+        if not revision:
+            self.fatal("Can't determine revision!")
+        self.revision = str(revision)
+        return self.revision
+
+    def _query_enUS_revision(self):
+        """Get revision from the objdir.
+        Only valid after setup is run.
+       """
+        if self.enUS_revision:
+            return self.enUS_revision
+        r = re.compile(r"^(gecko|fx)_revision ([0-9a-f]+\+?)$")
+        output = self._query_make_ident_output()
+        for line in output.splitlines():
+            match = r.match(line)
+            if match:
+                self.enUS_revision = match.groups()[1]
+        return self.enUS_revision
+
+    def _query_make_variable(self, variable, make_args=None,
+                             exclude_lines=PyMakeIgnoreList):
+        """returns the value of make echo-variable-<variable>
+           it accepts extra make arguements (make_args)
+           it also has an exclude_lines from the output filer
+           exclude_lines defaults to PyMakeIgnoreList because
+           on windows, pymake writes extra output lines that need
+           to be filtered out.
+        """
+        dirs = self.query_abs_dirs()
+        make_args = make_args or []
+        exclude_lines = exclude_lines or []
+        target = ["echo-variable-%s" % variable] + make_args
+        cwd = dirs['abs_locales_dir']
+        raw_output = self._get_output_from_make(target, cwd=cwd,
+                                                env=self.query_bootstrap_env())
+        # we want to log all the messages from make/pymake and
+        # exlcude some messages from the output ("Entering directory...")
+        output = []
+        for line in raw_output.split("\n"):
+            discard = False
+            for element in exclude_lines:
+                if element.match(line):
+                    discard = True
+                    continue
+            if not discard:
+                output.append(line.strip())
+        output = " ".join(output).strip()
+        self.info('echo-variable-%s: %s' % (variable, output))
+        return output
+
+    def query_version(self):
+        """Gets the version from the objdir.
+        Only valid after setup is run."""
+        if self.version:
+            return self.version
+        config = self.config
+        if config.get('release_config_file'):
+            release_config = self.query_release_config()
+            self.version = release_config['version']
+        else:
+            self.version = self._query_make_variable("MOZ_APP_VERSION")
+        return self.version
+
+    def _map(self, func, items):
+        """runs func for any item in items, calls the add_failure() for each
+           error. It assumes that function returns 0 when successful.
+           returns a two element tuple with (success_count, total_count)"""
+        success_count = 0
+        total_count = len(items)
+        name = func.__name__
+        for item in items:
+            result = func(item)
+            if result == SUCCESS:
+                #  success!
+                success_count += 1
+            else:
+                #  func failed...
+                message = 'failure: %s(%s)' % (name, item)
+                self._add_failure(item, message)
+        return (success_count, total_count)
+
+    def _add_failure(self, locale, message, **kwargs):
+        """marks current step as failed"""
+        self.locales_property[locale] = FAILURE_STR
+        prop_key = "%s_failure" % locale
+        prop_value = self.query_buildbot_property(prop_key)
+        if prop_value:
+            prop_value = "%s  %s" % (prop_value, message)
+        else:
+            prop_value = message
+        self.set_buildbot_property(prop_key, prop_value, write_to_file=True)
+        BaseScript.add_failure(self, locale, message=message, **kwargs)
+
+    def query_failed_locales(self):
+        return [l for l, res in self.locales_property.items() if
+                res == FAILURE_STR]
+
+    def summary(self):
+        """generates a summary"""
+        BaseScript.summary(self)
+        # TODO we probably want to make this configurable on/off
+        locales = self.query_locales()
+        for locale in locales:
+            self.locales_property.setdefault(locale, SUCCESS_STR)
+        self.set_buildbot_property("locales",
+                                   json.dumps(self.locales_property),
+                                   write_to_file=True)
+
+    # Actions {{{2
+    def clobber(self):
+        """clobber"""
+        dirs = self.query_abs_dirs()
+        clobber_dirs = (dirs['abs_objdir'], dirs['abs_upload_dir'])
+        PurgeMixin.clobber(self, always_clobber_dirs=clobber_dirs)
+
+    def pull(self):
+        """pulls source code"""
+        config = self.config
+        dirs = self.query_abs_dirs()
+        repos = []
+        # replace dictionary for repos
+        # we need to interpolate some values:
+        # branch, branch_repo
+        # and user_repo_override if exists
+        replace_dict = {}
+        if config.get("user_repo_override"):
+            replace_dict['user_repo_override'] = config['user_repo_override']
+        # this is OK so early because we get it from buildbot, or
+        # the command line for local dev
+        replace_dict['revision'] = self._query_revision()
+
+        for repository in config['repos']:
+            current_repo = {}
+            for key, value in repository.iteritems():
+                try:
+                    current_repo[key] = value % replace_dict
+                except TypeError:
+                    # pass through non-interpolables, like booleans
+                    current_repo[key] = value
+                except KeyError:
+                    self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value))
+                    raise
+            repos.append(current_repo)
+        self.info("repositories: %s" % repos)
+        self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
+                                tag_override=config.get('tag_override'))
+
+    def clone_locales(self):
+        self.pull_locale_source()
+
+    def setup(self):
+        """setup step"""
+        dirs = self.query_abs_dirs()
+        self._run_tooltool()
+        self._copy_mozconfig()
+        self._mach_configure()
+        self._run_make_in_config_dir()
+        self.make_wget_en_US()
+        self.make_unpack_en_US()
+        self.download_mar_tools()
+
+        # on try we want the source we already have, otherwise update to the
+        # same as the en-US binary
+        if self.config.get("update_gecko_source_to_enUS", True):
+            revision = self._query_enUS_revision()
+            #  TODO do this through VCSMixin instead of hardcoding hg
+            #  self.update(dest=dirs["abs_mozilla_dir"], revision=revision)
+            hg = self.query_exe("hg")
+            self.run_command([hg, "update", "-r", revision],
+                             cwd=dirs["abs_mozilla_dir"],
+                             env=self.query_bootstrap_env(),
+                             error_list=BaseErrorList,
+                             halt_on_failure=True, fatal_exit_code=3)
+            # if checkout updates CLOBBER file with a newer timestamp,
+            # next make -f client.mk configure  will delete archives
+            # downloaded with make wget_en_US, so just touch CLOBBER file
+            _clobber_file = self._clobber_file()
+            if os.path.exists(_clobber_file):
+                self._touch_file(_clobber_file)
+            # and again...
+            # thanks to the last hg update, we can be on different firefox 'version'
+            # than the one on default,
+            self._mach_configure()
+            self._run_make_in_config_dir()
+
+    def _run_make_in_config_dir(self):
+        """this step creates nsinstall, needed my make_wget_en_US()
+        """
+        dirs = self.query_abs_dirs()
+        config_dir = os.path.join(dirs['abs_objdir'], 'config')
+        env = self.query_bootstrap_env()
+        return self._make(target=['export'], cwd=config_dir, env=env)
+
+    def _clobber_file(self):
+        """returns the full path of the clobber file"""
+        config = self.config
+        dirs = self.query_abs_dirs()
+        return os.path.join(dirs['abs_objdir'], config.get('clobber_file'))
+
+    def _copy_mozconfig(self):
+        """copies the mozconfig file into abs_mozilla_dir/.mozconfig
+           and logs the content
+        """
+        config = self.config
+        dirs = self.query_abs_dirs()
+        mozconfig = config['mozconfig']
+        src = os.path.join(dirs['abs_work_dir'], mozconfig)
+        dst = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig')
+        self.copyfile(src, dst)
+        self.read_from_file(dst, verbose=True)
+
+    def _mach(self, target, env, halt_on_failure=True, output_parser=None):
+        dirs = self.query_abs_dirs()
+        mach = self._get_mach_executable()
+        return self.run_command(mach + target,
+                                halt_on_failure=True,
+                                env=env,
+                                cwd=dirs['abs_mozilla_dir'],
+                                output_parser=None)
+
+    def _mach_configure(self):
+        """calls mach configure"""
+        env = self.query_bootstrap_env()
+        target = ["configure"]
+        return self._mach(target=target, env=env)
+
+    def _get_mach_executable(self):
+        python = self.query_exe('python2.7')
+        return [python, 'mach']
+
+    def _get_make_executable(self):
+        config = self.config
+        dirs = self.query_abs_dirs()
+        if config.get('enable_mozmake'):  # e.g. windows
+            make = r"/".join([dirs['abs_mozilla_dir'], 'mozmake.exe'])
+            # mysterious subprocess errors, let's try to fix this path...
+            make = make.replace('\\', '/')
+            make = [make]
+        else:
+            make = ['make']
+        return make
+
+    def _make(self, target, cwd, env, error_list=MakefileErrorList,
+              halt_on_failure=True, output_parser=None):
+        """Runs make. Returns the exit code"""
+        make = self._get_make_executable()
+        if target:
+            make = make + target
+        return self.run_command(make,
+                                cwd=cwd,
+                                env=env,
+                                error_list=error_list,
+                                halt_on_failure=halt_on_failure,
+                                output_parser=output_parser)
+
+    def _get_output_from_make(self, target, cwd, env, halt_on_failure=True, ignore_errors=False):
+        """runs make and returns the output of the command"""
+        make = self._get_make_executable()
+        return self.get_output_from_command(make + target,
+                                            cwd=cwd,
+                                            env=env,
+                                            silent=True,
+                                            halt_on_failure=halt_on_failure,
+                                            ignore_errors=ignore_errors)
+
+    def make_unpack_en_US(self):
+        """wrapper for make unpack"""
+        config = self.config
+        dirs = self.query_abs_dirs()
+        env = self.query_bootstrap_env()
+        cwd = os.path.join(dirs['abs_objdir'], config['locales_dir'])
+        return self._make(target=["unpack"], cwd=cwd, env=env)
+
+    def make_wget_en_US(self):
+        """wrapper for make wget-en-US"""
+        env = self.query_bootstrap_env()
+        dirs = self.query_abs_dirs()
+        cwd = dirs['abs_locales_dir']
+        return self._make(target=["wget-en-US"], cwd=cwd, env=env)
+
+    def make_upload(self, locale):
+        """wrapper for make upload command"""
+        config = self.config
+        env = self.query_l10n_env()
+        dirs = self.query_abs_dirs()
+        buildid = self._query_buildid()
+        replace_dict = {
+            'buildid': buildid,
+            'branch': config['branch']
+        }
+        try:
+            env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict
+        except KeyError:
+            # no base_post_upload_cmd in configuration, just skip it
+            pass
+        target = ['upload', 'AB_CD=%s' % (locale)]
+        cwd = dirs['abs_locales_dir']
+        parser = MakeUploadOutputParser(config=self.config,
+                                        log_obj=self.log_obj)
+        retval = self._make(target=target, cwd=cwd, env=env,
+                            halt_on_failure=False, output_parser=parser)
+        if locale not in self.package_urls:
+            self.package_urls[locale] = {}
+        self.package_urls[locale].update(parser.matches)
+        if retval == SUCCESS:
+            self.info('Upload successful (%s)' % locale)
+            ret = SUCCESS
+        else:
+            self.error('failed to upload %s' % locale)
+            ret = FAILURE
+        return ret
+
+    def set_upload_files(self, locale):
+        # The tree doesn't have a good way of exporting the list of files
+        # created during locale generation, but we can grab them by echoing the
+        # UPLOAD_FILES variable for each locale.
+        env = self.query_l10n_env()
+        target = ['echo-variable-UPLOAD_FILES', 'echo-variable-CHECKSUM_FILES',
+                  'AB_CD=%s' % locale]
+        dirs = self.query_abs_dirs()
+        cwd = dirs['abs_locales_dir']
+        # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
+        #    we should ignore stderr as unfortunately it's expected when parsing for values
+        output = self._get_output_from_make(target=target, cwd=cwd, env=env,
+                                            ignore_errors=True)
+        self.info('UPLOAD_FILES is "%s"' % output)
+        files = shlex.split(output)
+        if not files:
+            self.error('failed to get upload file list for locale %s' % locale)
+            return FAILURE
+
+        self.upload_files[locale] = [
+            os.path.abspath(os.path.join(cwd, f)) for f in files
+        ]
+        return SUCCESS
+
+    def make_installers(self, locale):
+        """wrapper for make installers-(locale)"""
+        env = self.query_l10n_env()
+        self._copy_mozconfig()
+        dirs = self.query_abs_dirs()
+        cwd = os.path.join(dirs['abs_locales_dir'])
+        target = ["installers-%s" % locale,
+                  "LOCALE_MERGEDIR=%s" % env["LOCALE_MERGEDIR"], ]
+        return self._make(target=target, cwd=cwd,
+                          env=env, halt_on_failure=False)
+
+    def repack_locale(self, locale):
+        """wraps the logic for compare locale, make installers and generating
+           complete updates."""
+
+        if self.run_compare_locales(locale) != SUCCESS:
+            self.error("compare locale %s failed" % (locale))
+            return FAILURE
+
+        # compare locale succeeded, run make installers
+        if self.make_installers(locale) != SUCCESS:
+            self.error("make installers-%s failed" % (locale))
+            return FAILURE
+
+        # now try to upload the artifacts
+        if self.make_upload(locale):
+            self.error("make upload for locale %s failed!" % (locale))
+            return FAILURE
+
+        # set_upload_files() should be called after make upload, to make sure
+        # we have all files in place (checksums, etc)
+        if self.set_upload_files(locale):
+            self.error("failed to get list of files to upload for locale %s" % locale)
+            return FAILURE
+
+        return SUCCESS
+
+    def repack(self):
+        """creates the repacks and udpates"""
+        self._map(self.repack_locale, self.query_locales())
+
+    def _query_objdir(self):
+        """returns objdir name from configuration"""
+        return self.config['objdir']
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(DesktopSingleLocale, self).query_abs_dirs()
+        for directory in abs_dirs:
+            value = abs_dirs[directory]
+            abs_dirs[directory] = value
+        dirs = {}
+        dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
+        for key in dirs.keys():
+            if key not in abs_dirs:
+                abs_dirs[key] = dirs[key]
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def submit_to_balrog(self):
+        """submit to balrog"""
+        if not self.config.get("balrog_servers"):
+            self.info("balrog_servers not set; skipping balrog submission.")
+            return
+        self.info("Reading buildbot build properties...")
+        self.read_buildbot_config()
+        # get platform, appName and hashType from configuration
+        # common values across different locales
+        config = self.config
+        platform = config["platform"]
+        hashType = config['hashType']
+        appName = config['appName']
+        branch = config['branch']
+        # values from configuration
+        self.set_buildbot_property("branch", branch)
+        self.set_buildbot_property("appName", appName)
+        # it's hardcoded to sha512 in balrog.py
+        self.set_buildbot_property("hashType", hashType)
+        self.set_buildbot_property("platform", platform)
+        # values common to the current repacks
+        self.set_buildbot_property("buildid", self._query_buildid())
+        self.set_buildbot_property("appVersion", self.query_version())
+
+        # submit complete mar to balrog
+        # clean up buildbot_properties
+        self._map(self.submit_repack_to_balrog, self.query_locales())
+
+    def submit_repack_to_balrog(self, locale):
+        """submit a single locale to balrog"""
+        # check if locale has been uploaded, if not just return a FAILURE
+        if locale not in self.package_urls:
+            self.error("%s is not present in package_urls. Did you run make upload?" % locale)
+            return FAILURE
+
+        if not self.query_is_nightly():
+            # remove this check when we extend this script to non-nightly builds
+            self.fatal("Not a nightly build")
+            return FAILURE
+
+        # complete mar file
+        c_marfile = self._query_complete_mar_filename(locale)
+        c_mar_url = self._query_complete_mar_url(locale)
+
+        # Set other necessary properties for Balrog submission. None need to
+        # be passed back to buildbot, so we won't write them to the properties
+        # files
+        # Locale is hardcoded to en-US, for silly reasons
+        # The Balrog submitter translates this platform into a build target
+        # via https://github.com/mozilla/build-tools/blob/master/lib/python/release/platforms.py#L23
+        self.set_buildbot_property("completeMarSize", self.query_filesize(c_marfile))
+        self.set_buildbot_property("completeMarHash", self.query_sha512sum(c_marfile))
+        self.set_buildbot_property("completeMarUrl", c_mar_url)
+        self.set_buildbot_property("locale", locale)
+        if "partialInfo" in self.package_urls[locale]:
+            self.set_buildbot_property("partialInfo",
+                                       self.package_urls[locale]["partialInfo"])
+        ret = FAILURE
+        try:
+            result = self.submit_balrog_updates()
+            self.info("balrog return code: %s" % (result))
+            if result == 0:
+                ret = SUCCESS
+        except Exception as error:
+            self.error("submit repack to balrog failed: %s" % (str(error)))
+        return ret
+
+    def _query_complete_mar_filename(self, locale):
+        """returns the full path to a localized complete mar file"""
+        config = self.config
+        version = self.query_version()
+        complete_mar_name = config['localized_mar'] % {'version': version,
+                                                       'locale': locale}
+        return os.path.join(self._update_mar_dir(), complete_mar_name)
+
+    def _query_complete_mar_url(self, locale):
+        """returns the complete mar url taken from self.package_urls[locale]
+           this value is available only after make_upload"""
+        if "complete_mar_url" in self.config:
+            return self.config["complete_mar_url"]
+        if "completeMarUrl" in self.package_urls[locale]:
+            return self.package_urls[locale]["completeMarUrl"]
+        # url = self.config.get("update", {}).get("mar_base_url")
+        # if url:
+        #    url += os.path.basename(self.query_marfile_path())
+        #    return url.format(branch=self.query_branch())
+        self.fatal("Couldn't find complete mar url in config or package_urls")
+
+    def _update_mar_dir(self):
+        """returns the full path of the update/ directory"""
+        return self._mar_dir('update_mar_dir')
+
+    def _mar_binaries(self):
+        """returns a tuple with mar and mbsdiff paths"""
+        config = self.config
+        return (config['mar'], config['mbsdiff'])
+
+    def _mar_dir(self, dirname):
+        """returns the full path of dirname;
+            dirname is an entry in configuration"""
+        dirs = self.query_abs_dirs()
+        return os.path.join(dirs['abs_objdir'], self.config[dirname])
+
+    # TODO: replace with ToolToolMixin
+    def _get_tooltool_auth_file(self):
+        # set the default authentication file based on platform; this
+        # corresponds to where puppet puts the token
+        if 'tooltool_authentication_file' in self.config:
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
+        else:
+            fn = '/builds/relengapi.tok'
+
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
+
+    def _run_tooltool(self):
+        config = self.config
+        dirs = self.query_abs_dirs()
+        if not config.get('tooltool_manifest_src'):
+            return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
+        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
+                                         'scripts/tooltool/tooltool_wrapper.sh')
+        tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
+                                              config['tooltool_manifest_src'])
+        cmd = [
+            'sh',
+            fetch_script_path,
+            tooltool_manifest_path,
+            config['tooltool_url'],
+            config['tooltool_bootstrap'],
+        ]
+        cmd.extend(config['tooltool_script'])
+        auth_file = self._get_tooltool_auth_file()
+        if auth_file and os.path.exists(auth_file):
+            cmd.extend(['--authentication-file', auth_file])
+        cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
+        if cache:
+            cmd.extend(['-c', cache])
+        self.info(str(cmd))
+        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
+
+    def funsize_props(self):
+        """Set buildbot properties required to trigger funsize tasks
+         responsible to generate partial updates for successfully generated locales"""
+        locales = self.query_locales()
+        funsize_info = {
+            'locales': locales,
+            'branch': self.config['branch'],
+            'appName': self.config['appName'],
+            'platform': self.config['platform'],
+            'completeMarUrls':  {locale: self._query_complete_mar_url(locale) for locale in locales},
+        }
+        self.info('funsize info: %s' % funsize_info)
+        self.set_buildbot_property('funsize_info', json.dumps(funsize_info),
+                                   write_to_file=True)
+
+    def taskcluster_upload(self):
+        auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
+        credentials = {}
+        execfile(auth, credentials)
+        client_id = credentials.get('taskcluster_clientId')
+        access_token = credentials.get('taskcluster_accessToken')
+        if not client_id or not access_token:
+            self.warning('Skipping S3 file upload: No taskcluster credentials.')
+            return
+
+        # We need to activate the virtualenv so that we can import taskcluster
+        # (and its dependent modules, like requests and hawk).  Normally we
+        # could create the virtualenv as an action, but due to some odd
+        # dependencies with query_build_env() being called from build(), which
+        # is necessary before the virtualenv can be created.
+        self.disable_mock()
+        self.create_virtualenv()
+        self.enable_mock()
+        self.activate_virtualenv()
+
+        branch = self.config['branch']
+        revision = self._query_revision()
+        repo = self.query_l10n_repo()
+        if not repo:
+            self.fatal("Unable to determine repository for querying the push info.")
+        pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hg')
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
+
+        routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
+                                   'testing/mozharness/configs/routes.json')
+        with open(routes_json) as f:
+            contents = json.load(f)
+            templates = contents['l10n']
+
+        # Release promotion creates a special task to accumulate all artifacts
+        # under the same task
+        artifacts_task = None
+        self.read_buildbot_config()
+        if "artifactsTaskId" in self.buildbot_config.get("properties", {}):
+            artifacts_task_id = self.buildbot_config["properties"]["artifactsTaskId"]
+            artifacts_tc = Taskcluster(
+                    branch=branch, rank=pushinfo.pushdate, client_id=client_id,
+                    access_token=access_token, log_obj=self.log_obj,
+                    task_id=artifacts_task_id)
+            artifacts_task = artifacts_tc.get_task(artifacts_task_id)
+            artifacts_tc.claim_task(artifacts_task)
+
+        for locale, files in self.upload_files.iteritems():
+            self.info("Uploading files to S3 for locale '%s': %s" % (locale, files))
+            routes = []
+            for template in templates:
+                fmt = {
+                    'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
+                    'project': branch,
+                    'head_rev': revision,
+                    'pushdate': pushdate,
+                    'year': pushdate[0:4],
+                    'month': pushdate[4:6],
+                    'day': pushdate[6:8],
+                    'build_product': self.config['stage_product'],
+                    'build_name': self.query_build_name(),
+                    'build_type': self.query_build_type(),
+                    'locale': locale,
+                }
+                fmt.update(self.buildid_to_dict(self._query_buildid()))
+                routes.append(template.format(**fmt))
+
+            self.info('Using routes: %s' % routes)
+            tc = Taskcluster(branch,
+                             pushinfo.pushdate, # Use pushdate as the rank
+                             client_id,
+                             access_token,
+                             self.log_obj,
+                             )
+            task = tc.create_task(routes)
+            tc.claim_task(task)
+
+            for upload_file in files:
+                # Create an S3 artifact for each file that gets uploaded. We also
+                # check the uploaded file against the property conditions so that we
+                # can set the buildbot config with the correct URLs for package
+                # locations.
+                artifact_url = tc.create_artifact(task, upload_file)
+                if artifacts_task:
+                    artifacts_tc.create_reference_artifact(
+                            artifacts_task, upload_file, artifact_url)
+
+            tc.report_completed(task)
+
+        if artifacts_task:
+            if not self.query_failed_locales():
+                artifacts_tc.report_completed(artifacts_task)
+            else:
+                # If some locales fail, we want to mark the artifacts
+                # task failed, so a retry can reuse the same task ID
+                artifacts_tc.report_failed(artifacts_task)
+
+
+# main {{{
+if __name__ == '__main__':
+    single_locale = DesktopSingleLocale()
+    single_locale.run_and_exit()