summaryrefslogtreecommitdiffstats
path: root/testing/docker/recipes/run-task
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/docker/recipes/run-task
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/docker/recipes/run-task')
-rwxr-xr-xtesting/docker/recipes/run-task324
1 files changed, 324 insertions, 0 deletions
diff --git a/testing/docker/recipes/run-task b/testing/docker/recipes/run-task
new file mode 100755
index 000000000..978683cb5
--- /dev/null
+++ b/testing/docker/recipes/run-task
@@ -0,0 +1,324 @@
+#!/usr/bin/python2.7 -u
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Run a task after performing common actions.
+
+This script is meant to be the "driver" for TaskCluster based tasks.
+It receives some common arguments to control the run-time environment.
+
+It performs actions as requested from the arguments. Then it executes
+the requested process and prints its output, prefixing it with the
+current time to improve log usefulness.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import datetime
+import errno
+import grp
+import json
+import os
+import pwd
+import re
+import socket
+import stat
+import subprocess
+import sys
+import urllib2
+
+
+FINGERPRINT_URL = 'http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint'
+FALLBACK_FINGERPRINT = {
+ 'fingerprints':
+ "sha256:8e:ad:f7:6a:eb:44:06:15:ed:f3:e4:69:a6:64:60:37:2d:ff:98:88:37"
+ ":bf:d7:b8:40:84:01:48:9c:26:ce:d9"}
+
+
+def print_line(prefix, m):
+ now = datetime.datetime.utcnow()
+ print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m), end=b'')
+
+
+def run_and_prefix_output(prefix, args, extra_env=None):
+ """Runs a process and prefixes its output with the time.
+
+ Returns the process exit code.
+ """
+ print_line(prefix, b'executing %s\n' % args)
+
+ env = dict(os.environ)
+ env.update(extra_env or {})
+
+ # Note: TaskCluster's stdin is a TTY. This attribute is lost
+ # when we pass sys.stdin to the invoked process. If we cared
+ # to preserve stdin as a TTY, we could make this work. But until
+ # someone needs it, don't bother.
+ p = subprocess.Popen(args,
+ # Disable buffering because we want to receive output
+ # as it is generated so timestamps in logs are
+ # accurate.
+ bufsize=0,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ stdin=sys.stdin.fileno(),
+ cwd='/',
+ env=env,
+ # So \r in progress bars are rendered as multiple
+ # lines, preserving progress indicators.
+ universal_newlines=True)
+
+ while True:
+ data = p.stdout.readline()
+ if data == b'':
+ break
+
+ print_line(prefix, data)
+
+ return p.wait()
+
+
+def vcs_checkout(source_repo, dest, store_path,
+ base_repo=None, revision=None, branch=None):
+ # Specify method to checkout a revision. This defaults to revisions as
+ # SHA-1 strings, but also supports symbolic revisions like `tip` via the
+ # branch flag.
+ if revision:
+ revision_flag = b'--revision'
+ revision_value = revision
+ elif branch:
+ revision_flag = b'--branch'
+ revision_value = branch
+ else:
+ print('revision is not specified for checkout')
+ sys.exit(1)
+
+ # Obtain certificate fingerprints.
+ try:
+ print_line(b'vcs', 'fetching hg.mozilla.org fingerprint from %s\n' %
+ FINGERPRINT_URL)
+ res = urllib2.urlopen(FINGERPRINT_URL, timeout=10)
+ secret = res.read()
+ try:
+ secret = json.loads(secret, encoding='utf-8')
+ except ValueError:
+ print_line(b'vcs', 'invalid JSON in hg fingerprint secret')
+ sys.exit(1)
+ except (urllib2.URLError, socket.timeout):
+ print_line(b'vcs', 'Unable to retrieve current hg.mozilla.org fingerprint'
+ 'using the secret service, using fallback instead.')
+ # XXX This fingerprint will not be accurate if running on an old
+ # revision after the server fingerprint has changed.
+ secret = {'secret': FALLBACK_FINGERPRINT}
+
+ hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii')
+
+ args = [
+ b'/usr/bin/hg',
+ b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
+ b'robustcheckout',
+ b'--sharebase', store_path,
+ b'--purge',
+ ]
+
+ if base_repo:
+ args.extend([b'--upstream', base_repo])
+
+ args.extend([
+ revision_flag, revision_value,
+ source_repo, dest,
+ ])
+
+ res = run_and_prefix_output(b'vcs', args,
+ extra_env={b'PYTHONUNBUFFERED': b'1'})
+ if res:
+ sys.exit(res)
+
+ # Update the current revision hash and ensure that it is well formed.
+ revision = subprocess.check_output(
+ [b'/usr/bin/hg', b'log',
+ b'--rev', b'.',
+ b'--template', b'{node}'],
+ cwd=dest)
+
+ assert re.match('^[a-f0-9]{40}$', revision)
+ return revision
+
+
+def main(args):
+ print_line(b'setup', b'run-task started\n')
+
+ if os.getuid() != 0:
+ print('assertion failed: not running as root')
+ return 1
+
+ # Arguments up to '--' are ours. After are for the main task
+ # to be executed.
+ try:
+ i = args.index('--')
+ our_args = args[0:i]
+ task_args = args[i + 1:]
+ except ValueError:
+ our_args = args
+ task_args = []
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--user', default='worker', help='user to run as')
+ parser.add_argument('--group', default='worker', help='group to run as')
+ # We allow paths to be chowned by the --user:--group before permissions are
+ # dropped. This is often necessary for caches/volumes, since they default
+ # to root:root ownership.
+ parser.add_argument('--chown', action='append',
+ help='Directory to chown to --user:--group')
+ parser.add_argument('--chown-recursive', action='append',
+ help='Directory to recursively chown to --user:--group')
+ parser.add_argument('--vcs-checkout',
+ help='Directory where Gecko checkout should be created')
+ parser.add_argument('--tools-checkout',
+ help='Directory where build/tools checkout should be created')
+
+ args = parser.parse_args(our_args)
+
+ try:
+ user = pwd.getpwnam(args.user)
+ except KeyError:
+ print('could not find user %s; specify --user to a known user' %
+ args.user)
+ return 1
+ try:
+ group = grp.getgrnam(args.group)
+ except KeyError:
+ print('could not find group %s; specify --group to a known group' %
+ args.group)
+ return 1
+
+ uid = user.pw_uid
+ gid = group.gr_gid
+
+ # Find all groups to which this user is a member.
+ gids = [g.gr_gid for g in grp.getgrall() if args.group in g.gr_mem]
+
+ wanted_dir_mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
+
+ def set_dir_permissions(path, uid, gid):
+ st = os.lstat(path)
+
+ if st.st_uid != uid or st.st_gid != gid:
+ os.chown(path, uid, gid)
+
+ # Also make sure dirs are writable in case we need to delete
+ # them.
+ if st.st_mode & wanted_dir_mode != wanted_dir_mode:
+ os.chmod(path, st.st_mode | wanted_dir_mode)
+
+ # Change ownership of requested paths.
+ # FUTURE: parse argument values for user/group if we don't want to
+ # use --user/--group.
+ for path in args.chown or []:
+ print_line(b'chown', b'changing ownership of %s to %s:%s\n' % (
+ path, user.pw_name, group.gr_name))
+ set_dir_permissions(path, uid, gid)
+
+ for path in args.chown_recursive or []:
+ print_line(b'chown', b'recursively changing ownership of %s to %s:%s\n' %
+ (path, user.pw_name, group.gr_name))
+
+ set_dir_permissions(path, uid, gid)
+
+ for root, dirs, files in os.walk(path):
+ for d in dirs:
+ set_dir_permissions(os.path.join(root, d), uid, gid)
+
+ for f in files:
+ # File may be a symlink that points to nowhere. In which case
+ # os.chown() would fail because it attempts to follow the
+ # symlink. We only care about directory entries, not what
+ # they point to. So setting the owner of the symlink should
+ # be sufficient.
+ os.lchown(os.path.join(root, f), uid, gid)
+
+ def prepare_checkout_dir(checkout):
+ if not checkout:
+ return
+
+ # Ensure the directory for the source checkout exists.
+ try:
+ os.makedirs(os.path.dirname(checkout))
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ # And that it is owned by the appropriate user/group.
+ os.chown(os.path.dirname(checkout), uid, gid)
+
+ # And ensure the shared store path exists and has proper permissions.
+ if 'HG_STORE_PATH' not in os.environ:
+ print('error: HG_STORE_PATH environment variable not set')
+ sys.exit(1)
+
+ store_path = os.environ['HG_STORE_PATH']
+ try:
+ os.makedirs(store_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ os.chown(store_path, uid, gid)
+
+ prepare_checkout_dir(args.vcs_checkout)
+ prepare_checkout_dir(args.tools_checkout)
+
+ # Drop permissions to requested user.
+ # This code is modeled after what `sudo` was observed to do in a Docker
+ # container. We do not bother calling setrlimit() because containers have
+ # their own limits.
+ print_line(b'setup', b'running as %s:%s\n' % (args.user, args.group))
+ os.setgroups(gids)
+ os.umask(022)
+ os.setresgid(gid, gid, gid)
+ os.setresuid(uid, uid, uid)
+
+ # Checkout the repository, setting the GECKO_HEAD_REV to the current
+ # revision hash. Revision hashes have priority over symbolic revisions. We
+ # disallow running tasks with symbolic revisions unless they have been
+ # resolved by a checkout.
+ if args.vcs_checkout:
+ base_repo = os.environ.get('GECKO_BASE_REPOSITORY')
+ # Some callers set the base repository to mozilla-central for historical
+ # reasons. Switch to mozilla-unified because robustcheckout works best
+ # with it.
+ if base_repo == 'https://hg.mozilla.org/mozilla-central':
+ base_repo = b'https://hg.mozilla.org/mozilla-unified'
+
+ os.environ['GECKO_HEAD_REV'] = vcs_checkout(
+ os.environ['GECKO_HEAD_REPOSITORY'],
+ args.vcs_checkout,
+ os.environ['HG_STORE_PATH'],
+ base_repo=base_repo,
+ revision=os.environ.get('GECKO_HEAD_REV'),
+ branch=os.environ.get('GECKO_HEAD_REF'))
+
+ elif not os.environ.get('GECKO_HEAD_REV') and \
+ os.environ.get('GECKO_HEAD_REF'):
+ print('task should be defined in terms of non-symbolic revision')
+ return 1
+
+ if args.tools_checkout:
+ vcs_checkout(b'https://hg.mozilla.org/build/tools',
+ args.tools_checkout,
+ os.environ['HG_STORE_PATH'],
+ # Always check out the latest commit on default branch.
+ # This is non-deterministic!
+ branch=b'default')
+
+ return run_and_prefix_output(b'task', task_args)
+
+
+if __name__ == '__main__':
+ # Unbuffer stdio.
+ sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+ sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
+
+ sys.exit(main(sys.argv[1:]))