1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""fx_desktop_build.py.
script harness to build nightly firefox within Mozilla's build environment
and developer machines alike
author: Jordan Lund
"""
import copy
import pprint
import sys
import os
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
import mozharness.base.script as script
from mozharness.mozilla.building.buildbase import BUILD_BASE_CONFIG_OPTIONS, \
BuildingConfig, BuildOptionParser, BuildScript
from mozharness.base.config import parse_config_file
from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
class FxDesktopBuild(BuildScript, TryToolsMixin, object):
def __init__(self):
buildscript_kwargs = {
'config_options': BUILD_BASE_CONFIG_OPTIONS + copy.deepcopy(try_config_options),
'all_actions': [
'get-secrets',
'clobber',
'clone-tools',
'checkout-sources',
'setup-mock',
'build',
'upload-files', # upload from BB to TC
'sendchange',
'check-test',
'valgrind-test',
'package-source',
'generate-source-signing-manifest',
'multi-l10n',
'generate-build-stats',
'update',
],
'require_config_file': True,
# Default configuration
'config': {
'is_automation': True,
"pgo_build": False,
"debug_build": False,
"pgo_platforms": ['linux', 'linux64', 'win32', 'win64'],
# nightly stuff
"nightly_build": False,
'balrog_credentials_file': 'oauth.txt',
'taskcluster_credentials_file': 'oauth.txt',
'periodic_clobber': 168,
# hg tool stuff
"tools_repo": "https://hg.mozilla.org/build/tools",
# Seed all clones with mozilla-unified. This ensures subsequent
# jobs have a minimal `hg pull`.
"clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
"repo_base": "https://hg.mozilla.org",
'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
"graph_selector": "/server/collect.cgi",
# only used for make uploadsymbols
'old_packages': [
"%(objdir)s/dist/firefox-*",
"%(objdir)s/dist/fennec*",
"%(objdir)s/dist/seamonkey*",
"%(objdir)s/dist/thunderbird*",
"%(objdir)s/dist/install/sea/*.exe"
],
'stage_product': 'firefox',
'platform_supports_post_upload_to_latest': True,
'build_resources_path': '%(abs_src_dir)s/obj-firefox/.mozbuild/build_resources.json',
'nightly_promotion_branches': ['mozilla-central', 'mozilla-aurora'],
# try will overwrite these
'clone_with_purge': False,
'clone_by_revision': False,
'tinderbox_build_dir': None,
'to_tinderbox_dated': True,
'release_to_try_builds': False,
'include_post_upload_builddir': False,
'use_clobberer': True,
'stage_username': 'ffxbld',
'stage_ssh_key': 'ffxbld_rsa',
'virtualenv_modules': [
'requests==2.8.1',
'PyHawk-with-a-single-extra-commit==0.1.5',
'taskcluster==0.0.26',
],
'virtualenv_path': 'venv',
#
},
'ConfigClass': BuildingConfig,
}
super(FxDesktopBuild, self).__init__(**buildscript_kwargs)
def _pre_config_lock(self, rw_config):
"""grab buildbot props if we are running this in automation"""
super(FxDesktopBuild, self)._pre_config_lock(rw_config)
c = self.config
if c['is_automation']:
# parse buildbot config and add it to self.config
self.info("We are running this in buildbot, grab the build props")
self.read_buildbot_config()
###
if c.get('stage_platform'):
platform_for_log_url = c['stage_platform']
if c.get('pgo_build'):
platform_for_log_url += '-pgo'
# postrun.py uses stage_platform buildbot prop as part of the log url
self.set_buildbot_property('stage_platform',
platform_for_log_url,
write_to_file=True)
else:
self.fatal("'stage_platform' not determined and is required in your config")
if self.try_message_has_flag('artifact'):
self.info('Artifact build requested in try syntax.')
variant = 'artifact'
if c.get('build_variant') in ['debug', 'cross-debug']:
variant = 'debug-artifact'
self._update_build_variant(rw_config, variant)
# helpers
def _update_build_variant(self, rw_config, variant='artifact'):
""" Intended for use in _pre_config_lock """
c = self.config
variant_cfg_path, _ = BuildOptionParser.find_variant_cfg_path(
'--custom-build-variant-cfg',
variant,
rw_config.config_parser
)
if not variant_cfg_path:
self.fatal('Could not find appropriate config file for variant %s' % variant)
# Update other parts of config to keep dump-config accurate
# Only dump-config is affected because most config info is set during
# initial parsing
variant_cfg_dict = parse_config_file(variant_cfg_path)
rw_config.all_cfg_files_and_dicts.append((variant_cfg_path, variant_cfg_dict))
c.update({
'build_variant': variant,
'config_files': c['config_files'] + [variant_cfg_path]
})
self.info("Updating self.config with the following from {}:".format(variant_cfg_path))
self.info(pprint.pformat(variant_cfg_dict))
c.update(variant_cfg_dict)
c['forced_artifact_build'] = True
# Bug 1231320 adds MOZHARNESS_ACTIONS in TaskCluster tasks to override default_actions
# We don't want that when forcing an artifact build.
if rw_config.volatile_config['actions']:
self.info("Updating volatile_config to include default_actions "
"from {}.".format(variant_cfg_path))
# add default actions in correct order
combined_actions = []
for a in rw_config.all_actions:
if a in c['default_actions'] or a in rw_config.volatile_config['actions']:
combined_actions.append(a)
rw_config.volatile_config['actions'] = combined_actions
self.info("Actions in volatile_config are now: {}".format(
rw_config.volatile_config['actions'])
)
# replace rw_config as well to set actions as in BaseScript
rw_config.set_config(c, overwrite=True)
rw_config.update_actions()
self.actions = tuple(rw_config.actions)
self.all_actions = tuple(rw_config.all_actions)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
c = self.config
abs_dirs = super(FxDesktopBuild, self).query_abs_dirs()
if not c.get('app_ini_path'):
self.fatal('"app_ini_path" is needed in your config for this '
'script.')
dirs = {
# BuildFactories in factory.py refer to a 'build' dir on the slave.
# This contains all the source code/objdir to compile. However,
# there is already a build dir in mozharness for every mh run. The
# 'build' that factory refers to I named: 'src' so
# there is a seperation in mh. for example, rather than having
# '{mozharness_repo}/build/build/', I have '{
# mozharness_repo}/build/src/'
'abs_src_dir': os.path.join(abs_dirs['abs_work_dir'],
'src'),
'abs_obj_dir': os.path.join(abs_dirs['abs_work_dir'],
'src',
self._query_objdir()),
'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
'abs_app_ini_path': c['app_ini_path'] % {
'obj_dir': os.path.join(abs_dirs['abs_work_dir'],
'src',
self._query_objdir())
},
}
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
# Actions {{{2
# read_buildbot_config in BuildingMixin
# clobber in BuildingMixin -> PurgeMixin
# if Linux config:
# reset_mock in BuildingMixing -> MockMixin
# setup_mock in BuildingMixing (overrides MockMixin.mock_setup)
def set_extra_try_arguments(self, action, success=None):
""" Override unneeded method from TryToolsMixin """
pass
@script.PreScriptRun
def suppress_windows_modal_dialogs(self, *args, **kwargs):
if self._is_windows():
# Suppress Windows modal dialogs to avoid hangs
import ctypes
ctypes.windll.kernel32.SetErrorMode(0x8001)
if __name__ == '__main__':
fx_desktop_build = FxDesktopBuild()
fx_desktop_build.run_and_exit()
|
