summaryrefslogtreecommitdiffstats
path: root/testing/mozharness/scripts/spidermonkey_build.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/mozharness/scripts/spidermonkey_build.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/mozharness/scripts/spidermonkey_build.py')
-rwxr-xr-xtesting/mozharness/scripts/spidermonkey_build.py482
1 files changed, 482 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/spidermonkey_build.py b/testing/mozharness/scripts/spidermonkey_build.py
new file mode 100755
index 000000000..5522545da
--- /dev/null
+++ b/testing/mozharness/scripts/spidermonkey_build.py
@@ -0,0 +1,482 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import copy
+from datetime import datetime
+from functools import wraps
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import MakefileErrorList
+from mozharness.base.script import BaseScript
+from mozharness.base.transfer import TransferMixin
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.building.hazards import HazardError, HazardAnalysis
+from mozharness.mozilla.purge import PurgeMixin
+from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.tooltool import TooltoolMixin
+
+SUCCESS, WARNINGS, FAILURE, EXCEPTION, RETRY = xrange(5)
+
+
+def requires(*queries):
+ """Wrapper for detecting problems where some bit of information
+ required by the wrapped step is unavailable. Use it put prepending
+ @requires("foo"), which will check whether self.query_foo() returns
+ something useful."""
+ def make_wrapper(f):
+ @wraps(f)
+ def wrapper(self, *args, **kwargs):
+ for query in queries:
+ val = query(self)
+ goodval = not (val is None or "None" in str(val))
+ assert goodval, f.__name__ + " requires " + query.__name__ + " to return a value"
+ return f(self, *args, **kwargs)
+ return wrapper
+ return make_wrapper
+
+
+nuisance_env_vars = ['TERMCAP', 'LS_COLORS', 'PWD', '_']
+
+
+class SpidermonkeyBuild(MockMixin,
+ PurgeMixin, BaseScript,
+ VCSMixin, BuildbotMixin, TooltoolMixin, TransferMixin, BlobUploadMixin):
+ config_options = [
+ [["--repo"], {
+ "dest": "repo",
+ "help": "which gecko repo to get spidermonkey from",
+ }],
+ [["--source"], {
+ "dest": "source",
+ "help": "directory containing gecko source tree (instead of --repo)",
+ }],
+ [["--revision"], {
+ "dest": "revision",
+ }],
+ [["--branch"], {
+ "dest": "branch",
+ }],
+ [["--vcs-share-base"], {
+ "dest": "vcs_share_base",
+ "help": "base directory for shared repositories",
+ }],
+ [["-j"], {
+ "dest": "concurrency",
+ "type": int,
+ "default": 4,
+ "help": "number of simultaneous jobs used while building the shell " +
+ "(currently ignored for the analyzed build",
+ }] + copy.deepcopy(blobupload_config_options)
+ ]
+
+ def __init__(self):
+ super(SpidermonkeyBuild, self).__init__(
+ config_options=self.config_options,
+ # other stuff
+ all_actions=[
+ 'purge',
+ 'checkout-tools',
+
+ # First, build an optimized JS shell for running the analysis
+ 'checkout-source',
+ 'get-blobs',
+ 'clobber-shell',
+ 'configure-shell',
+ 'build-shell',
+
+ # Next, build a tree with the analysis plugin active. Note that
+ # we are using the same checkout for the JS shell build and the
+ # build of the source to be analyzed, which is a little
+ # unnecessary (no need to rebuild the JS shell all the time).
+ # (Different objdir, though.)
+
+ 'clobber-analysis',
+ 'setup-analysis',
+ 'run-analysis',
+ 'collect-analysis-output',
+ 'upload-analysis',
+ 'check-expectations',
+ ],
+ default_actions=[
+ 'purge',
+ 'checkout-tools',
+ 'checkout-source',
+ 'get-blobs',
+ 'clobber-shell',
+ 'configure-shell',
+ 'build-shell',
+ 'clobber-analysis',
+ 'setup-analysis',
+ 'run-analysis',
+ 'collect-analysis-output',
+ # Temporarily disabled, see bug 1211402
+ # 'upload-analysis',
+ 'check-expectations',
+ ],
+ config={
+ 'default_vcs': 'hg',
+ 'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
+ 'ccache': True,
+ 'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
+ 'tools_repo': 'https://hg.mozilla.org/build/tools',
+
+ 'upload_ssh_server': None,
+ 'upload_remote_basepath': None,
+ 'enable_try_uploads': True,
+ 'source': None,
+ 'stage_product': 'firefox',
+ },
+ )
+
+ self.buildid = None
+ self.create_virtualenv()
+ self.analysis = HazardAnalysis()
+
+ def _pre_config_lock(self, rw_config):
+ if self.config['source']:
+ self.config['srcdir'] = self.config['source']
+ super(SpidermonkeyBuild, self)._pre_config_lock(rw_config)
+
+ if self.buildbot_config is None:
+ self.info("Reading buildbot build properties...")
+ self.read_buildbot_config()
+
+ if self.buildbot_config:
+ bb_props = [('mock_target', 'mock_target', None),
+ ('hgurl', 'hgurl', None),
+ ('clobberer_url', 'clobberer_url', 'https://api.pub.build.mozilla.org/clobberer/lastclobber'),
+ ('force_clobber', 'force_clobber', None),
+ ('branch', 'blob_upload_branch', None),
+ ]
+ buildbot_props = self.buildbot_config.get('properties', {})
+ for bb_prop, cfg_prop, default in bb_props:
+ if not self.config.get(cfg_prop) and buildbot_props.get(bb_prop, default):
+ self.config[cfg_prop] = buildbot_props.get(bb_prop, default)
+ self.config['is_automation'] = True
+ else:
+ self.config['is_automation'] = False
+ self.config.setdefault('blob_upload_branch', 'devel')
+
+ dirs = self.query_abs_dirs()
+ replacements = self.config['env_replacements'].copy()
+ for k,v in replacements.items():
+ replacements[k] = v % dirs
+
+ self.env = self.query_env(replace_dict=replacements,
+ partial_env=self.config['partial_env'],
+ purge_env=nuisance_env_vars)
+ self.env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
+ self.env['TOOLTOOL_DIR'] = dirs['abs_work_dir']
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = BaseScript.query_abs_dirs(self)
+
+ abs_work_dir = abs_dirs['abs_work_dir']
+ dirs = {
+ 'shell_objdir':
+ os.path.join(abs_work_dir, self.config['shell-objdir']),
+ 'mozharness_scriptdir':
+ os.path.abspath(os.path.dirname(__file__)),
+ 'abs_analysis_dir':
+ os.path.join(abs_work_dir, self.config['analysis-dir']),
+ 'abs_analyzed_objdir':
+ os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
+ 'analysis_scriptdir':
+ os.path.join(self.config['srcdir'], self.config['analysis-scriptdir']),
+ 'abs_tools_dir':
+ os.path.join(abs_dirs['base_work_dir'], 'tools'),
+ 'gecko_src':
+ os.path.join(abs_work_dir, self.config['srcdir']),
+ 'abs_blob_upload_dir':
+ os.path.join(abs_work_dir, 'blobber_upload_dir'),
+ }
+
+ abs_dirs.update(dirs)
+ self.abs_dirs = abs_dirs
+
+ return self.abs_dirs
+
+ def query_repo(self):
+ if self.config.get('repo'):
+ return self.config['repo']
+ elif self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.config['hgurl'] + self.buildbot_config['properties']['repo_path']
+ else:
+ return None
+
+ def query_revision(self):
+ if 'revision' in self.buildbot_properties:
+ revision = self.buildbot_properties['revision']
+ elif self.buildbot_config and 'sourcestamp' in self.buildbot_config:
+ revision = self.buildbot_config['sourcestamp']['revision']
+ else:
+ # Useful for local testing. In actual use, this would always be
+ # None.
+ revision = self.config.get('revision')
+
+ return revision
+
+ def query_branch(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.buildbot_config['properties']['branch']
+ elif 'branch' in self.config:
+ # Used for locally testing try vs non-try
+ return self.config['branch']
+ else:
+ return os.path.basename(self.query_repo())
+
+ def query_compiler_manifest(self):
+ dirs = self.query_abs_dirs()
+ manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['compiler_manifest'])
+ if os.path.exists(manifest):
+ return manifest
+ return os.path.join(dirs['abs_work_dir'], self.config['compiler_manifest'])
+
+ def query_sixgill_manifest(self):
+ dirs = self.query_abs_dirs()
+ manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['sixgill_manifest'])
+ if os.path.exists(manifest):
+ return manifest
+ return os.path.join(dirs['abs_work_dir'], self.config['sixgill_manifest'])
+
+ def query_buildid(self):
+ if self.buildid:
+ return self.buildid
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ self.buildid = self.buildbot_config['properties'].get('buildid')
+ if not self.buildid:
+ self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
+ return self.buildid
+
+ def query_upload_ssh_server(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.buildbot_config['properties']['upload_ssh_server']
+ else:
+ return self.config['upload_ssh_server']
+
+ def query_upload_ssh_key(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ key = self.buildbot_config['properties']['upload_ssh_key']
+ else:
+ key = self.config['upload_ssh_key']
+ if self.mock_enabled and not key.startswith("/"):
+ key = "/home/mock_mozilla/.ssh/" + key
+ return key
+
+ def query_upload_ssh_user(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.buildbot_config['properties']['upload_ssh_user']
+ else:
+ return self.config['upload_ssh_user']
+
+ def query_product(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.buildbot_config['properties']['product']
+ else:
+ return self.config['product']
+
+ def query_upload_remote_basepath(self):
+ if self.config.get('upload_remote_basepath'):
+ return self.config['upload_remote_basepath']
+ else:
+ return "/pub/mozilla.org/{product}".format(
+ product=self.query_product(),
+ )
+
+ def query_upload_remote_baseuri(self):
+ baseuri = self.config.get('upload_remote_baseuri')
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ buildprops = self.buildbot_config['properties']
+ if 'upload_remote_baseuri' in buildprops:
+ baseuri = buildprops['upload_remote_baseuri']
+ return baseuri.strip("/") if baseuri else None
+
+ def query_target(self):
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ return self.buildbot_config['properties']['platform']
+ else:
+ return self.config.get('target')
+
+ def query_upload_path(self):
+ branch = self.query_branch()
+
+ common = {
+ 'basepath': self.query_upload_remote_basepath(),
+ 'branch': branch,
+ 'target': self.query_target(),
+ }
+
+ if branch == 'try':
+ if not self.config['enable_try_uploads']:
+ return None
+ try:
+ user = self.buildbot_config['sourcestamp']['changes'][0]['who']
+ except (KeyError, TypeError):
+ user = "unknown"
+ return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
+ user=user,
+ rev=self.query_revision(),
+ **common
+ )
+ else:
+ return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
+ buildid=self.query_buildid(),
+ **common
+ )
+
+ def query_do_upload(self):
+ if self.query_branch() == 'try':
+ return self.config.get('enable_try_uploads')
+ return True
+
+ # Actions {{{2
+ def purge(self):
+ dirs = self.query_abs_dirs()
+ self.info("purging, abs_upload_dir=" + dirs['abs_upload_dir'])
+ PurgeMixin.clobber(
+ self,
+ always_clobber_dirs=[
+ dirs['abs_upload_dir'],
+ ],
+ )
+
+ def checkout_tools(self):
+ dirs = self.query_abs_dirs()
+
+ # If running from within a directory also passed as the --source dir,
+ # this has the danger of clobbering <source>/tools/
+ if self.config['source']:
+ srcdir = self.config['source']
+ if os.path.samefile(srcdir, os.path.dirname(dirs['abs_tools_dir'])):
+ raise Exception("Cannot run from source checkout to avoid overwriting subdirs")
+
+ rev = self.vcs_checkout(
+ vcs='hg',
+ branch="default",
+ repo=self.config['tools_repo'],
+ clean=False,
+ dest=dirs['abs_tools_dir'],
+ )
+ self.set_buildbot_property("tools_revision", rev, write_to_file=True)
+
+ def do_checkout_source(self):
+ # --source option means to use an existing source directory instead of checking one out.
+ if self.config['source']:
+ return
+
+ dirs = self.query_abs_dirs()
+ dest = dirs['gecko_src']
+
+ # Pre-create the directory to appease the share extension
+ if not os.path.exists(dest):
+ self.mkdir_p(dest)
+
+ rev = self.vcs_checkout(
+ repo=self.query_repo(),
+ dest=dest,
+ revision=self.query_revision(),
+ branch=self.config.get('branch'),
+ clean=True,
+ )
+ self.set_buildbot_property('source_revision', rev, write_to_file=True)
+
+ def checkout_source(self):
+ try:
+ self.do_checkout_source()
+ except Exception as e:
+ self.fatal("checkout failed: " + str(e), exit_code=RETRY)
+
+ def get_blobs(self):
+ work_dir = self.query_abs_dirs()['abs_work_dir']
+ if not os.path.exists(work_dir):
+ self.mkdir_p(work_dir)
+ self.tooltool_fetch(self.query_compiler_manifest(), output_dir=work_dir)
+ self.tooltool_fetch(self.query_sixgill_manifest(), output_dir=work_dir)
+
+ def clobber_shell(self):
+ self.analysis.clobber_shell(self)
+
+ def configure_shell(self):
+ self.enable_mock()
+
+ try:
+ self.analysis.configure_shell(self)
+ except HazardError as e:
+ self.fatal(e, exit_code=FAILURE)
+
+ self.disable_mock()
+
+ def build_shell(self):
+ self.enable_mock()
+
+ try:
+ self.analysis.build_shell(self)
+ except HazardError as e:
+ self.fatal(e, exit_code=FAILURE)
+
+ self.disable_mock()
+
+ def clobber_analysis(self):
+ self.analysis.clobber(self)
+
+ def setup_analysis(self):
+ self.analysis.setup(self)
+
+ def run_analysis(self):
+ self.enable_mock()
+
+ upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
+ if not os.path.exists(upload_dir):
+ self.mkdir_p(upload_dir)
+
+ env = self.env.copy()
+ env['MOZ_UPLOAD_DIR'] = upload_dir
+
+ try:
+ self.analysis.run(self, env=env, error_list=MakefileErrorList)
+ except HazardError as e:
+ self.fatal(e, exit_code=FAILURE)
+
+ self.disable_mock()
+
+ def collect_analysis_output(self):
+ self.analysis.collect_output(self)
+
+ def upload_analysis(self):
+ if not self.config['is_automation']:
+ return
+
+ if not self.query_do_upload():
+ self.info("Uploads disabled for this build. Skipping...")
+ return
+
+ self.enable_mock()
+
+ try:
+ self.analysis.upload_results(self)
+ except HazardError as e:
+ self.error(e)
+ self.return_code = WARNINGS
+
+ self.disable_mock()
+
+ def check_expectations(self):
+ try:
+ self.analysis.check_expectations(self)
+ except HazardError as e:
+ self.fatal(e, exit_code=FAILURE)
+
+
+# main {{{1
+if __name__ == '__main__':
+ myScript = SpidermonkeyBuild()
+ myScript.run_and_exit()