diff options
Diffstat (limited to 'testing/mozharness/scripts/release')
10 files changed, 2056 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/release/antivirus.py b/testing/mozharness/scripts/release/antivirus.py new file mode 100644 index 000000000..b40dc5cc0 --- /dev/null +++ b/testing/mozharness/scripts/release/antivirus.py @@ -0,0 +1,193 @@ +from multiprocessing.pool import ThreadPool +import os +import re +import sys +import shutil + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript + + +class AntivirusScan(BaseScript, VirtualenvMixin): + config_options = [ + [["--product"], { + "dest": "product", + "help": "Product being released, eg: firefox, thunderbird", + }], + [["--version"], { + "dest": "version", + "help": "Version of release, eg: 39.0b5", + }], + [["--build-number"], { + "dest": "build_number", + "help": "Build number of release, eg: 2", + }], + [["--bucket-name"], { + "dest": "bucket_name", + "help": "S3 Bucket to retrieve files from", + }], + [["--exclude"], { + "dest": "excludes", + "action": "append", + "help": "List of filename patterns to exclude. See script source for default", + }], + [["-d", "--download-parallelization"], { + "dest": "download_parallelization", + "default": 6, + "type": "int", + "help": "Number of concurrent file downloads", + }], + [["-s", "--scan-parallelization"], { + "dest": "scan_parallelization", + "default": 4, + "type": "int", + "help": "Number of concurrent file scans", + }], + [["--tools-repo"], { + "dest": "tools_repo", + "default": "https://hg.mozilla.org/build/tools", + }], + [["--tools-revision"], { + "dest": "tools_revision", + "help": "Revision of tools repo to use when downloading extract_and_run_command.py", + }], + ] + virtualenv_config_options + + DEFAULT_EXCLUDES = [ + r"^.*tests.*$", + r"^.*crashreporter.*$", + r"^.*\.zip(\.asc)?$", + r"^.*\.log$", + r"^.*\.txt$", + r"^.*\.asc$", + r"^.*/partner-repacks.*$", + r"^.*.checksums(\.asc)?$", + r"^.*/logs/.*$", + r"^.*/jsshell.*$", + r"^.*json$", + r"^.*/host.*$", + r"^.*/mar-tools/.*$", + r"^.*robocop.apk$", + r"^.*contrib.*" + ] + CACHE_DIR = 'cache' + + def __init__(self): + BaseScript.__init__(self, + config_options=self.config_options, + require_config_file=False, + config={ + "virtualenv_modules": [ + "boto", + "redo", + "mar", + ], + "virtualenv_path": "venv", + }, + all_actions=[ + "create-virtualenv", + "activate-virtualenv", + "get-extract-script", + "get-files", + "scan-files", + "cleanup-cache", + ], + default_actions=[ + "create-virtualenv", + "activate-virtualenv", + "get-extract-script", + "get-files", + "scan-files", + "cleanup-cache", + ], + ) + self.excludes = self.config.get('excludes', self.DEFAULT_EXCLUDES) + self.dest_dir = self.CACHE_DIR + + def _get_candidates_prefix(self): + return "pub/{}/candidates/{}-candidates/build{}/".format( + self.config['product'], + self.config["version"], + self.config["build_number"] + ) + + def _matches_exclude(self, keyname): + for exclude in self.excludes: + if re.search(exclude, keyname): + return True + return False + + def get_extract_script(self): + """Gets a copy of extract_and_run_command.py from tools, and the supporting mar.py, + so that we can unpack various files for clam to scan them.""" + remote_file = "{}/raw-file/{}/stage/extract_and_run_command.py".format(self.config["tools_repo"], + self.config["tools_revision"]) + self.download_file(remote_file, file_name="extract_and_run_command.py") + + def get_files(self): + """Pull the candidate files down from S3 for scanning, using parallel requests""" + from boto.s3.connection import S3Connection + from boto.exception import S3CopyError, S3ResponseError + from redo import retry + from httplib import HTTPException + + # suppress boto debug logging, it's too verbose with --loglevel=debug + import logging + logging.getLogger('boto').setLevel(logging.INFO) + + self.info("Connecting to S3") + conn = S3Connection(anon=True) + self.info("Getting bucket {}".format(self.config["bucket_name"])) + bucket = conn.get_bucket(self.config["bucket_name"]) + + if os.path.exists(self.dest_dir): + self.info('Emptying {}'.format(self.dest_dir)) + shutil.rmtree(self.dest_dir) + os.makedirs(self.dest_dir) + + def worker(item): + source, destination = item + + self.info("Downloading {} to {}".format(source, destination)) + key = bucket.get_key(source) + return retry(key.get_contents_to_filename, + args=(destination, ), + sleeptime=30, max_sleeptime=150, + retry_exceptions=(S3CopyError, S3ResponseError, + IOError, HTTPException)) + + def find_release_files(): + candidates_prefix = self._get_candidates_prefix() + self.info("Getting key names from candidates") + for key in bucket.list(prefix=candidates_prefix): + keyname = key.name + if self._matches_exclude(keyname): + self.debug("Excluding {}".format(keyname)) + else: + destination = os.path.join(self.dest_dir, keyname.replace(candidates_prefix, '')) + dest_dir = os.path.dirname(destination) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + yield (keyname, destination) + + pool = ThreadPool(self.config["download_parallelization"]) + pool.map(worker, find_release_files()) + + def scan_files(self): + """Scan the files we've collected. We do the download and scan concurrently to make + it easier to have a coherent log afterwards. Uses the venv python.""" + self.run_command([self.query_python_path(), 'extract_and_run_command.py', + '-j{}'.format(self.config['scan_parallelization']), + 'clamdscan', '-m', '--no-summary', '--', self.dest_dir]) + + def cleanup_cache(self): + """If we have simultaneous releases in flight an av slave may end up doing another + av job before being recycled, and we need to make sure the full disk is available.""" + shutil.rmtree(self.dest_dir) + + +if __name__ == "__main__": + myScript = AntivirusScan() + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/release/beet_mover.py b/testing/mozharness/scripts/release/beet_mover.py new file mode 100755 index 000000000..adc8b19e1 --- /dev/null +++ b/testing/mozharness/scripts/release/beet_mover.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""beet_mover.py. + +downloads artifacts, scans them and uploads them to s3 +""" +import hashlib +import sys +import os +import pprint +import re +from os import listdir +from os.path import isfile, join +import sh +import redo + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) +from mozharness.base.log import FATAL +from mozharness.base.python import VirtualenvMixin +from mozharness.base.script import BaseScript +from mozharness.mozilla.aws import pop_aws_auth_from_env +import mozharness +import mimetypes + + +def get_hash(content, hash_type="md5"): + h = hashlib.new(hash_type) + h.update(content) + return h.hexdigest() + + +CONFIG_OPTIONS = [ + [["--template"], { + "dest": "template", + "help": "Specify jinja2 template file", + }], + [['--locale', ], { + "action": "extend", + "dest": "locales", + "type": "string", + "help": "Specify the locale(s) to upload."}], + [["--platform"], { + "dest": "platform", + "help": "Specify the platform of the build", + }], + [["--version"], { + "dest": "version", + "help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'" + }], + [["--app-version"], { + "dest": "app_version", + "help": "numbered version based on gecko. e.g. '44.0'" + }], + [["--partial-version"], { + "dest": "partial_version", + "help": "the partial version the mar is based off of" + }], + [["--artifact-subdir"], { + "dest": "artifact_subdir", + "default": 'build', + "help": "subdir location for taskcluster artifacts after public/ base.", + }], + [["--build-num"], { + "dest": "build_num", + "help": "the release build identifier" + }], + [["--taskid"], { + "dest": "taskid", + "help": "taskcluster task id to download artifacts from", + }], + [["--bucket"], { + "dest": "bucket", + "help": "s3 bucket to move beets to.", + }], + [["--product"], { + "dest": "product", + "help": "product for which artifacts are beetmoved", + }], + [["--exclude"], { + "dest": "excludes", + "action": "append", + "help": "List of filename patterns to exclude. See script source for default", + }], + [["-s", "--scan-parallelization"], { + "dest": "scan_parallelization", + "default": 4, + "type": "int", + "help": "Number of concurrent file scans", + }], +] + +DEFAULT_EXCLUDES = [ + r"^.*tests.*$", + r"^.*crashreporter.*$", + r"^.*\.zip(\.asc)?$", + r"^.*\.log$", + r"^.*\.txt$", + r"^.*\.asc$", + r"^.*/partner-repacks.*$", + r"^.*.checksums(\.asc)?$", + r"^.*/logs/.*$", + r"^.*/jsshell.*$", + r"^.*json$", + r"^.*/host.*$", + r"^.*/mar-tools/.*$", + r"^.*robocop.apk$", + r"^.*contrib.*" +] +CACHE_DIR = 'cache' + +MIME_MAP = { + '': 'text/plain', + '.asc': 'text/plain', + '.beet': 'text/plain', + '.bundle': 'application/octet-stream', + '.bz2': 'application/octet-stream', + '.checksums': 'text/plain', + '.dmg': 'application/x-iso9660-image', + '.mar': 'application/octet-stream', + '.xpi': 'application/x-xpinstall' +} + +HASH_FORMATS = ["sha512", "sha256"] + + +class BeetMover(BaseScript, VirtualenvMixin, object): + def __init__(self, aws_creds): + beetmover_kwargs = { + 'config_options': CONFIG_OPTIONS, + 'all_actions': [ + # 'clobber', + 'create-virtualenv', + 'activate-virtualenv', + 'generate-candidates-manifest', + 'refresh-antivirus', + 'verify-bits', # beets + 'download-bits', # beets + 'scan-bits', # beets + 'upload-bits', # beets + ], + 'require_config_file': False, + # Default configuration + 'config': { + # base index url where to find taskcluster artifact based on taskid + "artifact_base_url": 'https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/{subdir}', + "virtualenv_modules": [ + "boto", + "PyYAML", + "Jinja2", + "redo", + "cryptography==2.0.3", + "mar", + ], + "virtualenv_path": "venv", + }, + } + #todo do excludes need to be configured via command line for specific builds? + super(BeetMover, self).__init__(**beetmover_kwargs) + + c = self.config + self.manifest = {} + # assigned in _post_create_virtualenv + self.virtualenv_imports = None + self.bucket = c['bucket'] + if not all(aws_creds): + self.fatal('credentials must be passed in env: "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"') + self.aws_key_id, self.aws_secret_key = aws_creds + # if excludes is set from command line, use it otherwise use defaults + self.excludes = self.config.get('excludes', DEFAULT_EXCLUDES) + dirs = self.query_abs_dirs() + self.dest_dir = os.path.join(dirs['abs_work_dir'], CACHE_DIR) + self.mime_fix() + + def activate_virtualenv(self): + """ + activates virtualenv and adds module imports to a instance wide namespace. + + creating and activating a virtualenv onto the currently executing python interpreter is a + bit black magic. Rather than having import statements added in various places within the + script, we import them here immediately after we activate the newly created virtualenv + """ + VirtualenvMixin.activate_virtualenv(self) + + import boto + import yaml + import jinja2 + self.virtualenv_imports = { + 'boto': boto, + 'yaml': yaml, + 'jinja2': jinja2, + } + self.log("activated virtualenv with the modules: {}".format(str(self.virtualenv_imports))) + + def _get_template_vars(self): + return { + "platform": self.config['platform'], + "locales": self.config.get('locales'), + "version": self.config['version'], + "app_version": self.config.get('app_version', ''), + "partial_version": self.config.get('partial_version', ''), + "build_num": self.config['build_num'], + # keep the trailing slash + "s3_prefix": 'pub/{prod}/candidates/{ver}-candidates/{n}/'.format( + prod=self.config['product'], ver=self.config['version'], + n=self.config['build_num'] + ), + "artifact_base_url": self.config['artifact_base_url'].format( + taskid=self.config['taskid'], subdir=self.config['artifact_subdir'] + ) + } + + def generate_candidates_manifest(self): + """ + generates and outputs a manifest that maps expected Taskcluster artifact names + to release deliverable names + """ + self.log('generating manifest from {}...'.format(self.config['template'])) + template_dir, template_file = os.path.split(os.path.abspath(self.config['template'])) + jinja2 = self.virtualenv_imports['jinja2'] + yaml = self.virtualenv_imports['yaml'] + + jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir), + undefined=jinja2.StrictUndefined) + template = jinja_env.get_template(template_file) + self.manifest = yaml.safe_load(template.render(**self._get_template_vars())) + + self.log("manifest generated:") + self.log(pprint.pformat(self.manifest['mapping'])) + + def verify_bits(self): + """ + inspects each artifact and verifies that they were created by trustworthy tasks + """ + # TODO + self.log('skipping verification. unimplemented...') + + def refresh_antivirus(self): + self.info("Refreshing clamav db...") + try: + redo.retry(lambda: + sh.freshclam("--stdout", "--verbose", _timeout=300, + _err_to_out=True)) + self.info("Done.") + except sh.ErrorReturnCode: + self.warning("Freshclam failed, skipping DB update") + + def download_bits(self): + """ + downloads list of artifacts to self.dest_dir dir based on a given manifest + """ + self.log('downloading and uploading artifacts to self_dest_dir...') + dirs = self.query_abs_dirs() + + for locale in self.manifest['mapping']: + for deliverable in self.manifest['mapping'][locale]: + self.log("downloading '{}' deliverable for '{}' locale".format(deliverable, locale)) + source = self.manifest['mapping'][locale][deliverable]['artifact'] + self.retry( + self.download_file, + args=[source], + kwargs={'parent_dir': dirs['abs_work_dir']}, + error_level=FATAL) + self.log('Success!') + + def _strip_prefix(self, s3_key): + """Return file name relative to prefix""" + # "abc/def/hfg".split("abc/de")[-1] == "f/hfg" + return s3_key.split(self._get_template_vars()["s3_prefix"])[-1] + + def upload_bits(self): + """ + uploads list of artifacts to s3 candidates dir based on a given manifest + """ + self.log('uploading artifacts to s3...') + dirs = self.query_abs_dirs() + + # connect to s3 + boto = self.virtualenv_imports['boto'] + conn = boto.connect_s3(self.aws_key_id, self.aws_secret_key) + bucket = conn.get_bucket(self.bucket) + + for locale in self.manifest['mapping']: + for deliverable in self.manifest['mapping'][locale]: + self.log("uploading '{}' deliverable for '{}' locale".format(deliverable, locale)) + # we have already downloaded the files locally so we can use that version + source = self.manifest['mapping'][locale][deliverable]['artifact'] + s3_key = self.manifest['mapping'][locale][deliverable]['s3_key'] + downloaded_file = os.path.join(dirs['abs_work_dir'], self.get_filename_from_url(source)) + # generate checksums for every uploaded file + beet_file_name = '{}.beet'.format(downloaded_file) + # upload checksums to a separate subdirectory + beet_dest = '{prefix}beetmover-checksums/{f}.beet'.format( + prefix=self._get_template_vars()["s3_prefix"], + f=self._strip_prefix(s3_key) + ) + beet_contents = '\n'.join([ + '{hash} {fmt} {size} {name}'.format( + hash=self.get_hash_for_file(downloaded_file, hash_type=fmt), + fmt=fmt, + size=os.path.getsize(downloaded_file), + name=self._strip_prefix(s3_key)) for fmt in HASH_FORMATS + ]) + self.write_to_file(beet_file_name, beet_contents) + self.upload_bit(source=downloaded_file, s3_key=s3_key, + bucket=bucket) + self.upload_bit(source=beet_file_name, s3_key=beet_dest, + bucket=bucket) + self.log('Success!') + + + def upload_bit(self, source, s3_key, bucket): + boto = self.virtualenv_imports['boto'] + self.info('uploading to s3 with key: {}'.format(s3_key)) + key = boto.s3.key.Key(bucket) # create new key + key.key = s3_key # set key name + + self.info("Checking if `{}` already exists".format(s3_key)) + key = bucket.get_key(s3_key) + if not key: + self.info("Uploading to `{}`".format(s3_key)) + key = bucket.new_key(s3_key) + # set key value + mime_type, _ = mimetypes.guess_type(source) + self.retry(lambda: key.set_contents_from_filename(source, headers={'Content-Type': mime_type}), + error_level=FATAL), + else: + if not get_hash(key.get_contents_as_string()) == get_hash(open(source).read()): + # for now, let's halt. If necessary, we can revisit this and allow for overwrites + # to the same buildnum release with different bits + self.fatal("`{}` already exists with different checksum.".format(s3_key)) + self.log("`{}` has the same MD5 checksum, not uploading".format(s3_key)) + + def scan_bits(self): + + dirs = self.query_abs_dirs() + + filenames = [f for f in listdir(dirs['abs_work_dir']) if isfile(join(dirs['abs_work_dir'], f))] + self.mkdir_p(self.dest_dir) + for file_name in filenames: + if self._matches_exclude(file_name): + self.info("Excluding {} from virus scan".format(file_name)) + else: + self.info('Copying {} to {}'.format(file_name,self.dest_dir)) + self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name)) + self._scan_files() + self.info('Emptying {}'.format(self.dest_dir)) + self.rmtree(self.dest_dir) + + def _scan_files(self): + """Scan the files we've collected. We do the download and scan concurrently to make + it easier to have a coherent log afterwards. Uses the venv python.""" + external_tools_path = os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools') + self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'), + '-j{}'.format(self.config['scan_parallelization']), + 'clamscan', '--no-summary', '--', self.dest_dir]) + + def _matches_exclude(self, keyname): + return any(re.search(exclude, keyname) for exclude in self.excludes) + + def mime_fix(self): + """ Add mimetypes for custom extensions """ + mimetypes.init() + map(lambda (ext, mime_type,): mimetypes.add_type(mime_type, ext), MIME_MAP.items()) + +if __name__ == '__main__': + beet_mover = BeetMover(pop_aws_auth_from_env()) + beet_mover.run_and_exit() diff --git a/testing/mozharness/scripts/release/generate-checksums.py b/testing/mozharness/scripts/release/generate-checksums.py new file mode 100644 index 000000000..61a1c43d2 --- /dev/null +++ b/testing/mozharness/scripts/release/generate-checksums.py @@ -0,0 +1,284 @@ +from multiprocessing.pool import ThreadPool +import os +from os import path +import re +import sys +import posixpath + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.base.vcs.vcsbase import VCSMixin +from mozharness.mozilla.checksums import parse_checksums_file +from mozharness.mozilla.signing import SigningMixin +from mozharness.mozilla.buildbot import BuildbotMixin + +class ChecksumsGenerator(BaseScript, VirtualenvMixin, SigningMixin, VCSMixin, BuildbotMixin): + config_options = [ + [["--stage-product"], { + "dest": "stage_product", + "help": "Name of product used in file server's directory structure, eg: firefox, mobile", + }], + [["--version"], { + "dest": "version", + "help": "Version of release, eg: 39.0b5", + }], + [["--build-number"], { + "dest": "build_number", + "help": "Build number of release, eg: 2", + }], + [["--bucket-name-prefix"], { + "dest": "bucket_name_prefix", + "help": "Prefix of bucket name, eg: net-mozaws-prod-delivery. This will be used to generate a full bucket name (such as net-mozaws-prod-delivery-{firefox,archive}.", + }], + [["--bucket-name-full"], { + "dest": "bucket_name_full", + "help": "Full bucket name, eg: net-mozaws-prod-delivery-firefox", + }], + [["-j", "--parallelization"], { + "dest": "parallelization", + "default": 20, + "type": int, + "help": "Number of checksums file to download concurrently", + }], + [["-f", "--format"], { + "dest": "formats", + "default": [], + "action": "append", + "help": "Format(s) to generate big checksums file for. Default: sha512", + }], + [["--include"], { + "dest": "includes", + "default": [], + "action": "append", + "help": "List of patterns to include in big checksums file. See script source for default.", + }], + [["--tools-repo"], { + "dest": "tools_repo", + "default": "https://hg.mozilla.org/build/tools", + }], + [["--credentials"], { + "dest": "credentials", + "help": "File containing access key and secret access key for S3", + }], + ] + virtualenv_config_options + + def __init__(self): + BaseScript.__init__(self, + config_options=self.config_options, + require_config_file=False, + config={ + "virtualenv_modules": [ + "pip==1.5.5", + "boto", + ], + "virtualenv_path": "venv", + 'buildbot_json_path': 'buildprops.json', + }, + all_actions=[ + "create-virtualenv", + "collect-individual-checksums", + "create-big-checksums", + "sign", + "upload", + "copy-info-files", + ], + default_actions=[ + "create-virtualenv", + "collect-individual-checksums", + "create-big-checksums", + "sign", + "upload", + ], + ) + + self.checksums = {} + self.bucket = None + self.bucket_name = self._get_bucket_name() + self.file_prefix = self._get_file_prefix() + # set the env var for boto to read our special config file + # rather than anything else we have at ~/.boto + os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"]) + + def _pre_config_lock(self, rw_config): + super(ChecksumsGenerator, self)._pre_config_lock(rw_config) + + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + # TODO: version should come from repo + props = self.buildbot_config["properties"] + for prop in ['version', 'build_number']: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + + # These defaults are set here rather in the config because default + # lists cannot be completely overidden, only appended to. + if not self.config.get("formats"): + self.config["formats"] = ["sha512", "sha256"] + + if not self.config.get("includes"): + self.config["includes"] = [ + r"^.*\.tar\.bz2$", + r"^.*\.tar\.xz$", + r"^.*\.dmg$", + r"^.*\.bundle$", + r"^.*\.mar$", + r"^.*Setup.*\.exe$", + r"^.*\.xpi$", + r"^.*fennec.*\.apk$", + ] + + def _get_bucket_name(self): + if self.config.get('bucket_name_full'): + return self.config['bucket_name_full'] + + suffix = "archive" + # Firefox has a special bucket, per https://github.com/mozilla-services/product-delivery-tools/blob/master/bucketmap.go + if self.config["stage_product"] == "firefox": + suffix = "firefox" + + return "{}-{}".format(self.config["bucket_name_prefix"], suffix) + + def _get_file_prefix(self): + return "pub/{}/candidates/{}-candidates/build{}/".format( + self.config["stage_product"], self.config["version"], self.config["build_number"] + ) + + def _get_sums_filename(self, format_): + return "{}SUMS".format(format_.upper()) + + def _get_bucket(self): + if not self.bucket: + self.activate_virtualenv() + from boto.s3.connection import S3Connection + + self.info("Connecting to S3") + conn = S3Connection() + self.debug("Successfully connected to S3") + self.info("Connecting to bucket {}".format(self.bucket_name)) + self.bucket = conn.get_bucket(self.bucket_name) + return self.bucket + + def collect_individual_checksums(self): + """This step grabs all of the small checksums files for the release, + filters out any unwanted files from within them, and adds the remainder + to self.checksums for subsequent steps to use.""" + bucket = self._get_bucket() + self.info("File prefix is: {}".format(self.file_prefix)) + + # Temporary holding place for checksums + raw_checksums = [] + def worker(item): + self.debug("Downloading {}".format(item)) + # TODO: It would be nice to download the associated .asc file + # and verify against it. + sums = bucket.get_key(item).get_contents_as_string() + raw_checksums.append(sums) + + def find_checksums_files(): + self.info("Getting key names from bucket") + checksum_files = {"beets": [], "checksums": []} + for key in bucket.list(prefix=self.file_prefix): + if key.key.endswith(".checksums"): + self.debug("Found checksums file: {}".format(key.key)) + checksum_files["checksums"].append(key.key) + elif key.key.endswith(".beet"): + self.debug("Found beet file: {}".format(key.key)) + checksum_files["beets"].append(key.key) + else: + self.debug("Ignoring non-checksums file: {}".format(key.key)) + if checksum_files["beets"]: + self.log("Using beet format") + return checksum_files["beets"] + else: + self.log("Using checksums format") + return checksum_files["checksums"] + + pool = ThreadPool(self.config["parallelization"]) + pool.map(worker, find_checksums_files()) + + for c in raw_checksums: + for f, info in parse_checksums_file(c).iteritems(): + for pattern in self.config["includes"]: + if re.search(pattern, f): + if f in self.checksums: + self.fatal("Found duplicate checksum entry for {}, don't know which one to pick.".format(f)) + if not set(self.config["formats"]) <= set(info["hashes"]): + self.fatal("Missing necessary format for file {}".format(f)) + self.debug("Adding checksums for file: {}".format(f)) + self.checksums[f] = info + break + else: + self.debug("Ignoring checksums for file: {}".format(f)) + + def create_big_checksums(self): + for fmt in self.config["formats"]: + sums = self._get_sums_filename(fmt) + self.info("Creating big checksums file: {}".format(sums)) + with open(sums, "w+") as output_file: + for fn in sorted(self.checksums): + output_file.write("{} {}\n".format(self.checksums[fn]["hashes"][fmt], fn)) + + def sign(self): + dirs = self.query_abs_dirs() + + tools_dir = path.join(dirs["abs_work_dir"], "tools") + self.vcs_checkout( + repo=self.config["tools_repo"], + branch="default", + vcs="hg", + dest=tools_dir, + ) + + sign_cmd = self.query_moz_sign_cmd(formats=["gpg"]) + + for fmt in self.config["formats"]: + sums = self._get_sums_filename(fmt) + self.info("Signing big checksums file: {}".format(sums)) + retval = self.run_command(sign_cmd + [sums]) + if retval != 0: + self.fatal("Failed to sign {}".format(sums)) + + def upload(self): + # we need to provide the public side of the gpg key so that people can + # verify the detached signatures + dirs = self.query_abs_dirs() + tools_dir = path.join(dirs["abs_work_dir"], "tools") + self.copyfile(os.path.join(tools_dir, 'scripts', 'release', 'KEY'), + 'KEY') + files = ['KEY'] + + for fmt in self.config["formats"]: + files.append(self._get_sums_filename(fmt)) + files.append("{}.asc".format(self._get_sums_filename(fmt))) + + bucket = self._get_bucket() + for f in files: + dest = posixpath.join(self.file_prefix, f) + self.info("Uploading {} to {}".format(f, dest)) + key = bucket.new_key(dest) + key.set_contents_from_filename(f, headers={'Content-Type': 'text/plain'}) + + def copy_info_files(self): + bucket = self._get_bucket() + + for key in bucket.list(prefix=self.file_prefix): + if re.search(r'/en-US/android.*_info\.txt$', key.name): + self.info("Found {}".format(key.name)) + dest = posixpath.join(self.file_prefix, posixpath.basename(key.name)) + self.info("Copying to {}".format(dest)) + bucket.copy_key(new_key_name=dest, + src_bucket_name=self.bucket_name, + src_key_name=key.name, + metadata={'Content-Type': 'text/plain'}) + + +if __name__ == "__main__": + myScript = ChecksumsGenerator() + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py b/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py new file mode 100644 index 000000000..78a60b4bc --- /dev/null +++ b/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" postrelease_bouncer_aliases.py + +A script to replace the old-fashion way of updating the bouncer aliaes through +tools script. +""" + +import os +import sys + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.mozilla.buildbot import BuildbotMixin + + +# PostReleaseBouncerAliases {{{1 +class PostReleaseBouncerAliases(BaseScript, VirtualenvMixin, BuildbotMixin): + config_options = virtualenv_config_options + + def __init__(self, require_config_file=True): + super(PostReleaseBouncerAliases, self).__init__( + config_options=self.config_options, + require_config_file=require_config_file, + config={ + "virtualenv_modules": [ + "redo", + "requests", + ], + "virtualenv_path": "venv", + 'credentials_file': 'oauth.txt', + 'buildbot_json_path': 'buildprops.json', + }, + all_actions=[ + "create-virtualenv", + "activate-virtualenv", + "update-bouncer-aliases", + ], + default_actions=[ + "create-virtualenv", + "activate-virtualenv", + "update-bouncer-aliases", + ], + ) + + def _pre_config_lock(self, rw_config): + super(PostReleaseBouncerAliases, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + props = self.buildbot_config["properties"] + for prop in ['tuxedo_server_url', 'version']: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + else: + self.warning("%s could not be found within buildprops" % prop) + return + + def _update_bouncer_alias(self, tuxedo_server_url, auth, + related_product, alias): + from redo import retry + import requests + + url = "%s/create_update_alias" % tuxedo_server_url + data = {"alias": alias, "related_product": related_product} + self.log("Updating {} to point to {} using {}".format(alias, + related_product, + url)) + + # Wrap the real call to hide credentials from retry's logging + def do_update_bouncer_alias(): + r = requests.post(url, data=data, auth=auth, + verify=False, timeout=60) + r.raise_for_status() + + retry(do_update_bouncer_alias) + + def update_bouncer_aliases(self): + tuxedo_server_url = self.config['tuxedo_server_url'] + credentials_file = os.path.join(os.getcwd(), + self.config['credentials_file']) + credentials = {} + execfile(credentials_file, credentials) + auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword']) + version = self.config['version'] + for product, info in self.config["products"].iteritems(): + if "alias" in info: + product_template = info["product-name"] + related_product = product_template % {"version": version} + self._update_bouncer_alias(tuxedo_server_url, auth, + related_product, info["alias"]) + + +# __main__ {{{1 +if __name__ == '__main__': + PostReleaseBouncerAliases().run_and_exit() diff --git a/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py b/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py new file mode 100644 index 000000000..f84b5771c --- /dev/null +++ b/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" postrelease_mark_as_shipped.py + +A script to automate the manual way of updating a release as shipped in Ship-it +following its successful ship-to-the-door opertion. +""" +import os +import sys +from datetime import datetime + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.mozilla.buildbot import BuildbotMixin + + +def build_release_name(product, version, buildno): + """Function to reconstruct the name of the release based on product, + version and buildnumber + """ + return "{}-{}-build{}".format(product.capitalize(), + str(version), str(buildno)) + + +class MarkReleaseAsShipped(BaseScript, VirtualenvMixin, BuildbotMixin): + config_options = virtualenv_config_options + + def __init__(self, require_config_file=True): + super(MarkReleaseAsShipped, self).__init__( + config_options=self.config_options, + require_config_file=require_config_file, + config={ + "virtualenv_modules": [ + "shipitapi", + ], + "virtualenv_path": "venv", + "credentials_file": "oauth.txt", + "buildbot_json_path": "buildprops.json", + "timeout": 60, + }, + all_actions=[ + "create-virtualenv", + "activate-virtualenv", + "mark-as-shipped", + ], + default_actions=[ + "create-virtualenv", + "activate-virtualenv", + "mark-as-shipped", + ], + ) + + def _pre_config_lock(self, rw_config): + super(MarkReleaseAsShipped, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + props = self.buildbot_config['properties'] + mandatory_props = ['product', 'version', 'build_number'] + missing_props = [] + for prop in mandatory_props: + if prop in props: + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + else: + self.warning("%s could not be found within buildprops" % prop) + missing_props.append(prop) + + if missing_props: + raise Exception("%s not found in configs" % missing_props) + + self.config['name'] = build_release_name(self.config['product'], + self.config['version'], + self.config['build_number']) + + def mark_as_shipped(self): + """Method to make a simple call to Ship-it API to change a release + status to 'shipped' + """ + credentials_file = os.path.join(os.getcwd(), + self.config["credentials_file"]) + credentials = {} + execfile(credentials_file, credentials) + ship_it_credentials = credentials["ship_it_credentials"] + auth = (self.config["ship_it_username"], + ship_it_credentials.get(self.config["ship_it_username"])) + api_root = self.config['ship_it_root'] + + from shipitapi import Release + release_api = Release(auth, api_root=api_root, + timeout=self.config['timeout']) + shipped_at = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + + self.info("Mark the release as shipped with %s timestamp" % shipped_at) + release_api.update(self.config['name'], + status='shipped', shippedAt=shipped_at) + + +if __name__ == '__main__': + MarkReleaseAsShipped().run_and_exit() diff --git a/testing/mozharness/scripts/release/postrelease_version_bump.py b/testing/mozharness/scripts/release/postrelease_version_bump.py new file mode 100644 index 000000000..dfffa699a --- /dev/null +++ b/testing/mozharness/scripts/release/postrelease_version_bump.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" postrelease_version_bump.py + +A script to increase in-tree version number after shipping a release. +""" + +import os +import sys + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.buildbot import BuildbotMixin +from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin + + +# PostReleaseVersionBump {{{1 +class PostReleaseVersionBump(MercurialScript, BuildbotMixin, + MercurialRepoManipulationMixin): + config_options = [ + [['--hg-user', ], { + "action": "store", + "dest": "hg_user", + "type": "string", + "default": "ffxbld <release@mozilla.com>", + "help": "Specify what user to use to commit to hg.", + }], + [['--next-version', ], { + "action": "store", + "dest": "next_version", + "type": "string", + "help": "Next version used in version bump", + }], + [['--ssh-user', ], { + "action": "store", + "dest": "ssh_user", + "type": "string", + "help": "SSH username with hg.mozilla.org permissions", + }], + [['--ssh-key', ], { + "action": "store", + "dest": "ssh_key", + "type": "string", + "help": "Path to SSH key.", + }], + [['--product', ], { + "action": "store", + "dest": "product", + "type": "string", + "help": "Product name", + }], + [['--version', ], { + "action": "store", + "dest": "version", + "type": "string", + "help": "Version", + }], + [['--build-number', ], { + "action": "store", + "dest": "build_number", + "type": "string", + "help": "Build number", + }], + [['--revision', ], { + "action": "store", + "dest": "revision", + "type": "string", + "help": "HG revision to tag", + }], + ] + + def __init__(self, require_config_file=True): + super(PostReleaseVersionBump, self).__init__( + config_options=self.config_options, + all_actions=[ + 'clobber', + 'clean-repos', + 'pull', + 'bump_postrelease', + 'commit-changes', + 'tag', + 'push', + ], + default_actions=[ + 'clean-repos', + 'pull', + 'bump_postrelease', + 'commit-changes', + 'tag', + 'push', + ], + config={ + 'buildbot_json_path': 'buildprops.json', + }, + require_config_file=require_config_file + ) + + def _pre_config_lock(self, rw_config): + super(PostReleaseVersionBump, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + else: + props = self.buildbot_config["properties"] + for prop in ['next_version', 'product', 'version', 'build_number', + 'revision']: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + + if not self.config.get("next_version"): + self.fatal("Next version has to be set. Use --next-version or " + "pass `next_version' via buildbot properties.") + + def query_abs_dirs(self): + """ Allow for abs_from_dir and abs_to_dir + """ + if self.abs_dirs: + return self.abs_dirs + self.abs_dirs = super(PostReleaseVersionBump, self).query_abs_dirs() + self.abs_dirs["abs_gecko_dir"] = os.path.join( + self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"]) + return self.abs_dirs + + def query_repos(self): + """Build a list of repos to clone.""" + return [self.config["repo"]] + + def query_commit_dirs(self): + return [self.query_abs_dirs()["abs_gecko_dir"]] + + def query_commit_message(self): + return "Automatic version bump. CLOSED TREE NO BUG a=release" + + def query_push_dirs(self): + return self.query_commit_dirs() + + def query_push_args(self, cwd): + # cwd is not used here + hg_ssh_opts = "ssh -l {user} -i {key}".format( + user=self.config["ssh_user"], + key=os.path.expanduser(self.config["ssh_key"]) + ) + return ["-e", hg_ssh_opts, "-r", "."] + + def pull(self): + super(PostReleaseVersionBump, self).pull( + repos=self.query_repos()) + + def bump_postrelease(self, *args, **kwargs): + """Bump version""" + dirs = self.query_abs_dirs() + for f in self.config["version_files"]: + curr_version = ".".join( + self.get_version(dirs['abs_gecko_dir'], f["file"])) + self.replace(os.path.join(dirs['abs_gecko_dir'], f["file"]), + curr_version, self.config["next_version"]) + + def tag(self): + dirs = self.query_abs_dirs() + tags = ["{product}_{version}_BUILD{build_number}", + "{product}_{version}_RELEASE"] + tags = [t.format(product=self.config["product"].upper(), + version=self.config["version"].replace(".", "_"), + build_number=self.config["build_number"]) + for t in tags] + message = "No bug - Tagging {revision} with {tags} a=release CLOSED TREE" + message = message.format( + revision=self.config["revision"], + tags=', '.join(tags)) + self.hg_tag(cwd=dirs["abs_gecko_dir"], tags=tags, + revision=self.config["revision"], message=message, + user=self.config["hg_user"], force=True) + +# __main__ {{{1 +if __name__ == '__main__': + PostReleaseVersionBump().run_and_exit() diff --git a/testing/mozharness/scripts/release/publish_balrog.py b/testing/mozharness/scripts/release/publish_balrog.py new file mode 100644 index 000000000..edb381311 --- /dev/null +++ b/testing/mozharness/scripts/release/publish_balrog.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" updates.py + +A script publish a release to Balrog. + +""" + +import os +import sys + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.buildbot import BuildbotMixin + +# PublishBalrog {{{1 + + +class PublishBalrog(MercurialScript, BuildbotMixin): + + def __init__(self, require_config_file=True): + super(PublishBalrog, self).__init__( + all_actions=[ + 'clobber', + 'pull', + 'submit-to-balrog', + ], + default_actions=[ + 'clobber', + 'pull', + 'submit-to-balrog', + ], + config={ + 'buildbot_json_path': 'buildprops.json', + 'credentials_file': 'oauth.txt', + }, + require_config_file=require_config_file + ) + + def _pre_config_lock(self, rw_config): + super(PublishBalrog, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + # TODO: version and appVersion should come from repo + props = self.buildbot_config["properties"] + for prop in ['product', 'version', 'build_number', 'channels', + 'balrog_api_root', 'schedule_at', 'background_rate']: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + self.abs_dirs = super(PublishBalrog, self).query_abs_dirs() + self.abs_dirs["abs_tools_dir"] = os.path.join( + self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"]) + return self.abs_dirs + + def query_channel_configs(self): + """Return a list of channel configs. + For RC builds it returns "release" and "beta" using + "enabled_if_version_matches" to match RC. + + :return: list + """ + return [(n, c) for n, c in self.config["update_channels"].items() if + n in self.config["channels"]] + + def query_repos(self): + """Build a list of repos to clone.""" + return [self.config["repo"]] + + def pull(self): + super(PublishBalrog, self).pull( + repos=self.query_repos()) + + + def submit_to_balrog(self): + for _, channel_config in self.query_channel_configs(): + self._submit_to_balrog(channel_config) + + def _submit_to_balrog(self, channel_config): + dirs = self.query_abs_dirs() + auth = os.path.join(os.getcwd(), self.config['credentials_file']) + cmd = [ + self.query_exe("python"), + os.path.join(dirs["abs_tools_dir"], + "scripts/build-promotion/balrog-release-shipper.py")] + cmd.extend([ + "--api-root", self.config["balrog_api_root"], + "--credentials-file", auth, + "--username", self.config["balrog_username"], + "--version", self.config["version"], + "--product", self.config["product"], + "--build-number", str(self.config["build_number"]), + "--verbose", + ]) + for r in channel_config["publish_rules"]: + cmd.extend(["--rules", r]) + if self.config.get("schedule_at"): + cmd.extend(["--schedule-at", self.config["schedule_at"]]) + if self.config.get("background_rate"): + cmd.extend(["--background-rate", str(self.config["background_rate"])]) + + self.retry(lambda: self.run_command(cmd, halt_on_failure=True)) + +# __main__ {{{1 +if __name__ == '__main__': + PublishBalrog().run_and_exit() diff --git a/testing/mozharness/scripts/release/push-candidate-to-releases.py b/testing/mozharness/scripts/release/push-candidate-to-releases.py new file mode 100644 index 000000000..5339fa38a --- /dev/null +++ b/testing/mozharness/scripts/release/push-candidate-to-releases.py @@ -0,0 +1,200 @@ +from multiprocessing.pool import ThreadPool +import os +import re +import sys + + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.mozilla.aws import pop_aws_auth_from_env + + +class ReleasePusher(BaseScript, VirtualenvMixin): + config_options = [ + [["--product"], { + "dest": "product", + "help": "Product being released, eg: firefox, thunderbird", + }], + [["--version"], { + "dest": "version", + "help": "Version of release, eg: 39.0b5", + }], + [["--build-number"], { + "dest": "build_number", + "help": "Build number of release, eg: 2", + }], + [["--bucket-name"], { + "dest": "bucket_name", + "help": "Bucket to copy files from candidates/ to releases/", + }], + [["--credentials"], { + "dest": "credentials", + "help": "File containing access key and secret access key", + }], + [["--exclude"], { + "dest": "excludes", + "default": [ + r"^.*tests.*$", + r"^.*crashreporter.*$", + r"^.*[^k]\.zip(\.asc)?$", + r"^.*\.log$", + r"^.*\.txt$", + r"^.*/partner-repacks.*$", + r"^.*.checksums(\.asc)?$", + r"^.*/logs/.*$", + r"^.*/jsshell.*$", + r"^.*json$", + r"^.*/host.*$", + r"^.*/mar-tools/.*$", + r"^.*robocop.apk$", + r"^.*bouncer.apk$", + r"^.*contrib.*", + r"^.*/beetmover-checksums/.*$", + ], + "action": "append", + "help": "List of patterns to exclude from copy. The list can be " + "extended by passing multiple --exclude arguments.", + }], + [["-j", "--parallelization"], { + "dest": "parallelization", + "default": 20, + "type": "int", + "help": "Number of copy requests to run concurrently", + }], + ] + virtualenv_config_options + + def __init__(self, aws_creds): + BaseScript.__init__(self, + config_options=self.config_options, + require_config_file=False, + config={ + "virtualenv_modules": [ + "boto", + "redo", + ], + "virtualenv_path": "venv", + }, + all_actions=[ + "create-virtualenv", + "activate-virtualenv", + "push-to-releases", + ], + default_actions=[ + "create-virtualenv", + "activate-virtualenv", + "push-to-releases", + ], + ) + + # validate aws credentials + if not (all(aws_creds) or self.config.get('credentials')): + self.fatal("aws creds not defined. please add them to your config or env.") + if any(aws_creds) and self.config.get('credentials'): + self.fatal("aws creds found in env and self.config. please declare in one place only.") + + # set aws credentials + if all(aws_creds): + self.aws_key_id, self.aws_secret_key = aws_creds + else: # use + self.aws_key_id, self.aws_secret_key = None, None + # set the env var for boto to read our special config file + # rather than anything else we have at ~/.boto + os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"]) + + def _get_candidates_prefix(self): + return "pub/{}/candidates/{}-candidates/build{}/".format( + self.config['product'], + self.config["version"], + self.config["build_number"] + ) + + def _get_releases_prefix(self): + return "pub/{}/releases/{}/".format( + self.config["product"], + self.config["version"] + ) + + def _matches_exclude(self, keyname): + for exclude in self.config["excludes"]: + if re.search(exclude, keyname): + return True + return False + + def push_to_releases(self): + """This step grabs the list of files in the candidates dir, + filters out any unwanted files from within them, and copies + the remainder.""" + from boto.s3.connection import S3Connection + from boto.exception import S3CopyError, S3ResponseError + from redo import retry + + # suppress boto debug logging, it's too verbose with --loglevel=debug + import logging + logging.getLogger('boto').setLevel(logging.INFO) + + self.info("Connecting to S3") + conn = S3Connection(aws_access_key_id=self.aws_key_id, + aws_secret_access_key=self.aws_secret_key) + self.info("Getting bucket {}".format(self.config["bucket_name"])) + bucket = conn.get_bucket(self.config["bucket_name"]) + + # ensure the destination is empty + self.info("Checking destination {} is empty".format(self._get_releases_prefix())) + keys = [k for k in bucket.list(prefix=self._get_releases_prefix())] + if keys: + self.warning("Destination already exists with %s keys" % len(keys)) + + def worker(item): + source, destination = item + + def copy_key(): + source_key = bucket.get_key(source) + dest_key = bucket.get_key(destination) + # According to http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html + # S3 key MD5 is represented as ETag, except when objects are + # uploaded using multipart method. In this case objects's ETag + # is constructed using its MD5, minus symbol, and number of + # part. See http://stackoverflow.com/questions/12186993/what-is-the-algorithm-to-compute-the-amazon-s3-etag-for-a-file-larger-than-5gb#answer-19896823 + source_md5 = source_key.etag.split("-")[0] + if dest_key: + dest_md5 = dest_key.etag.split("-")[0] + else: + dest_md5 = None + + if not dest_key: + self.info("Copying {} to {}".format(source, destination)) + bucket.copy_key(destination, self.config["bucket_name"], + source) + elif source_md5 == dest_md5: + self.warning( + "{} already exists with the same content ({}), skipping copy".format( + destination, dest_md5)) + else: + self.fatal( + "{} already exists with the different content (src ETag: {}, dest ETag: {}), aborting".format( + destination, source_key.etag, dest_key.etag)) + + return retry(copy_key, sleeptime=5, max_sleeptime=60, + retry_exceptions=(S3CopyError, S3ResponseError)) + + def find_release_files(): + candidates_prefix = self._get_candidates_prefix() + release_prefix = self._get_releases_prefix() + self.info("Getting key names from candidates") + for key in bucket.list(prefix=candidates_prefix): + keyname = key.name + if self._matches_exclude(keyname): + self.debug("Excluding {}".format(keyname)) + else: + destination = keyname.replace(candidates_prefix, + release_prefix) + yield (keyname, destination) + + pool = ThreadPool(self.config["parallelization"]) + pool.map(worker, find_release_files()) + +if __name__ == "__main__": + myScript = ReleasePusher(pop_aws_auth_from_env()) + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/release/updates.py b/testing/mozharness/scripts/release/updates.py new file mode 100644 index 000000000..4b660a67b --- /dev/null +++ b/testing/mozharness/scripts/release/updates.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" updates.py + +A script to bump patcher configs, generate update verification configs, and +publish top-level release blob information to Balrog. + +It clones the tools repo, modifies the existing patcher config to include +current release build information, generates update verification configs, +commits the changes and tags the repo using tags by Releng convention. +After the changes are pushed to the repo, the script submits top-level release +information to Balrog. +""" + +import os +import re +import sys + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.buildbot import BuildbotMixin +from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin +from mozharness.mozilla.release import get_previous_version + + +# UpdatesBumper {{{1 +class UpdatesBumper(MercurialScript, BuildbotMixin, + MercurialRepoManipulationMixin): + config_options = [ + [['--hg-user', ], { + "action": "store", + "dest": "hg_user", + "type": "string", + "default": "ffxbld <release@mozilla.com>", + "help": "Specify what user to use to commit to hg.", + }], + [['--ssh-user', ], { + "action": "store", + "dest": "ssh_user", + "type": "string", + "help": "SSH username with hg.mozilla.org permissions", + }], + [['--ssh-key', ], { + "action": "store", + "dest": "ssh_key", + "type": "string", + "help": "Path to SSH key.", + }], + ] + + def __init__(self, require_config_file=True): + super(UpdatesBumper, self).__init__( + config_options=self.config_options, + all_actions=[ + 'clobber', + 'pull', + 'download-shipped-locales', + 'bump-configs', + 'commit-changes', + 'tag', + 'push', + 'submit-to-balrog', + ], + default_actions=[ + 'clobber', + 'pull', + 'download-shipped-locales', + 'bump-configs', + 'commit-changes', + 'tag', + 'push', + 'submit-to-balrog', + ], + config={ + 'buildbot_json_path': 'buildprops.json', + 'credentials_file': 'oauth.txt', + }, + require_config_file=require_config_file + ) + + def _pre_config_lock(self, rw_config): + super(UpdatesBumper, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + # TODO: version and appVersion should come from repo + props = self.buildbot_config["properties"] + for prop in ['product', 'version', 'build_number', 'revision', + 'appVersion', 'balrog_api_root', "channels"]: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + + partials = [v.strip() for v in props["partial_versions"].split(",")] + self.config["partial_versions"] = [v.split("build") for v in partials] + self.config["platforms"] = [p.strip() for p in + props["platforms"].split(",")] + self.config["channels"] = [c.strip() for c in + props["channels"].split(",")] + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + self.abs_dirs = super(UpdatesBumper, self).query_abs_dirs() + self.abs_dirs["abs_tools_dir"] = os.path.join( + self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"]) + return self.abs_dirs + + def query_repos(self): + """Build a list of repos to clone.""" + return [self.config["repo"]] + + def query_commit_dirs(self): + return [self.query_abs_dirs()["abs_tools_dir"]] + + def query_commit_message(self): + return "Automated configuration bump" + + def query_push_dirs(self): + return self.query_commit_dirs() + + def query_push_args(self, cwd): + # cwd is not used here + hg_ssh_opts = "ssh -l {user} -i {key}".format( + user=self.config["ssh_user"], + key=os.path.expanduser(self.config["ssh_key"]) + ) + return ["-e", hg_ssh_opts] + + def query_shipped_locales_path(self): + dirs = self.query_abs_dirs() + return os.path.join(dirs["abs_work_dir"], "shipped-locales") + + def query_channel_configs(self): + """Return a list of channel configs. + For RC builds it returns "release" and "beta" using + "enabled_if_version_matches" to match RC. + + :return: list + """ + return [(n, c) for n, c in self.config["update_channels"].items() if + n in self.config["channels"]] + + def pull(self): + super(UpdatesBumper, self).pull( + repos=self.query_repos()) + + def download_shipped_locales(self): + dirs = self.query_abs_dirs() + self.mkdir_p(dirs["abs_work_dir"]) + url = self.config["shipped-locales-url"].format( + revision=self.config["revision"]) + if not self.download_file(url=url, + file_name=self.query_shipped_locales_path()): + self.fatal("Unable to fetch shipped-locales from %s" % url) + + def bump_configs(self): + for channel, channel_config in self.query_channel_configs(): + self.bump_patcher_config(channel_config) + self.bump_update_verify_configs(channel, channel_config) + + def query_matching_partials(self, channel_config): + return [(v, b) for v, b in self.config["partial_versions"] if + re.match(channel_config["version_regex"], v)] + + def query_patcher_config(self, channel_config): + dirs = self.query_abs_dirs() + patcher_config = os.path.join( + dirs["abs_tools_dir"], "release/patcher-configs", + channel_config["patcher_config"]) + return patcher_config + + def query_update_verify_config(self, channel, platform): + dirs = self.query_abs_dirs() + uvc = os.path.join( + dirs["abs_tools_dir"], "release/updates", + "{}-{}-{}.cfg".format(channel, self.config["product"], platform)) + return uvc + + def bump_patcher_config(self, channel_config): + # TODO: to make it possible to run this before we have files copied to + # the candidates directory, we need to add support to fetch build IDs + # from tasks. + dirs = self.query_abs_dirs() + env = {"PERL5LIB": os.path.join(dirs["abs_tools_dir"], "lib/perl")} + partial_versions = [v[0] for v in + self.query_matching_partials(channel_config)] + script = os.path.join( + dirs["abs_tools_dir"], "release/patcher-config-bump.pl") + patcher_config = self.query_patcher_config(channel_config) + cmd = [self.query_exe("perl"), script] + cmd.extend([ + "-p", self.config["product"], + "-r", self.config["product"].capitalize(), + "-v", self.config["version"], + "-a", self.config["appVersion"], + "-o", get_previous_version( + self.config["version"], partial_versions), + "-b", str(self.config["build_number"]), + "-c", patcher_config, + "-f", self.config["archive_domain"], + "-d", self.config["download_domain"], + "-l", self.query_shipped_locales_path(), + ]) + for v in partial_versions: + cmd.extend(["--partial-version", v]) + for p in self.config["platforms"]: + cmd.extend(["--platform", p]) + for mar_channel_id in channel_config["mar_channel_ids"]: + cmd.extend(["--mar-channel-id", mar_channel_id]) + self.run_command(cmd, halt_on_failure=True, env=env) + + def bump_update_verify_configs(self, channel, channel_config): + dirs = self.query_abs_dirs() + script = os.path.join( + dirs["abs_tools_dir"], + "scripts/build-promotion/create-update-verify-config.py") + patcher_config = self.query_patcher_config(channel_config) + for platform in self.config["platforms"]: + cmd = [self.query_exe("python"), script] + output = self.query_update_verify_config(channel, platform) + cmd.extend([ + "--config", patcher_config, + "--platform", platform, + "--update-verify-channel", + channel_config["update_verify_channel"], + "--output", output, + "--archive-prefix", self.config["archive_prefix"], + "--previous-archive-prefix", + self.config["previous_archive_prefix"], + "--product", self.config["product"], + "--balrog-url", self.config["balrog_url"], + "--build-number", str(self.config["build_number"]), + ]) + + self.run_command(cmd, halt_on_failure=True) + + def tag(self): + dirs = self.query_abs_dirs() + tags = ["{product}_{version}_BUILD{build_number}_RUNTIME", + "{product}_{version}_RELEASE_RUNTIME"] + tags = [t.format(product=self.config["product"].upper(), + version=self.config["version"].replace(".", "_"), + build_number=self.config["build_number"]) + for t in tags] + self.hg_tag(cwd=dirs["abs_tools_dir"], tags=tags, + user=self.config["hg_user"], force=True) + + def submit_to_balrog(self): + for _, channel_config in self.query_channel_configs(): + self._submit_to_balrog(channel_config) + + def _submit_to_balrog(self, channel_config): + dirs = self.query_abs_dirs() + auth = os.path.join(os.getcwd(), self.config['credentials_file']) + cmd = [ + self.query_exe("python"), + os.path.join(dirs["abs_tools_dir"], + "scripts/build-promotion/balrog-release-pusher.py")] + cmd.extend([ + "--api-root", self.config["balrog_api_root"], + "--download-domain", self.config["download_domain"], + "--archive-domain", self.config["archive_domain"], + "--credentials-file", auth, + "--product", self.config["product"], + "--version", self.config["version"], + "--build-number", str(self.config["build_number"]), + "--app-version", self.config["appVersion"], + "--username", self.config["balrog_username"], + "--verbose", + ]) + for c in channel_config["channel_names"]: + cmd.extend(["--channel", c]) + for r in channel_config["rules_to_update"]: + cmd.extend(["--rule-to-update", r]) + for p in self.config["platforms"]: + cmd.extend(["--platform", p]) + for v, build_number in self.query_matching_partials(channel_config): + partial = "{version}build{build_number}".format( + version=v, build_number=build_number) + cmd.extend(["--partial-update", partial]) + if channel_config["requires_mirrors"]: + cmd.append("--requires-mirrors") + if self.config["balrog_use_dummy_suffix"]: + cmd.append("--dummy") + + self.retry(lambda: self.run_command(cmd, halt_on_failure=True)) + +# __main__ {{{1 +if __name__ == '__main__': + UpdatesBumper().run_and_exit() diff --git a/testing/mozharness/scripts/release/uptake_monitoring.py b/testing/mozharness/scripts/release/uptake_monitoring.py new file mode 100644 index 000000000..9ec24621f --- /dev/null +++ b/testing/mozharness/scripts/release/uptake_monitoring.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# lint_ignore=E501 +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" uptake_monitoring.py + +A script to replace the old-fashion way of computing the uptake monitoring +from the scheduler within the slaves. +""" + +import os +import sys +import datetime +import time +import xml.dom.minidom + +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import VirtualenvMixin, virtualenv_config_options +from mozharness.base.script import BaseScript +from mozharness.mozilla.buildbot import BuildbotMixin + + +def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os): + return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url, + related_product, os) + + +class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin): + config_options = virtualenv_config_options + + def __init__(self, require_config_file=True): + super(UptakeMonitoring, self).__init__( + config_options=self.config_options, + require_config_file=require_config_file, + config={ + "virtualenv_modules": [ + "redo", + "requests", + ], + + "virtualenv_path": "venv", + "credentials_file": "oauth.txt", + "buildbot_json_path": "buildprops.json", + "poll_interval": 60, + "poll_timeout": 20*60, + "min_uptake": 10000, + }, + all_actions=[ + "create-virtualenv", + "activate-virtualenv", + "monitor-uptake", + ], + default_actions=[ + "create-virtualenv", + "activate-virtualenv", + "monitor-uptake", + ], + ) + + def _pre_config_lock(self, rw_config): + super(UptakeMonitoring, self)._pre_config_lock(rw_config) + # override properties from buildbot properties here as defined by + # taskcluster properties + self.read_buildbot_config() + if not self.buildbot_config: + self.warning("Skipping buildbot properties overrides") + return + props = self.buildbot_config["properties"] + for prop in ['tuxedo_server_url', 'version']: + if props.get(prop): + self.info("Overriding %s with %s" % (prop, props[prop])) + self.config[prop] = props.get(prop) + else: + self.warning("%s could not be found within buildprops" % prop) + return + partials = [v.strip() for v in props["partial_versions"].split(",")] + self.config["partial_versions"] = [v.split("build")[0] for v in partials] + self.config["platforms"] = [p.strip() for p in + props["platforms"].split(",")] + + def _get_product_uptake(self, tuxedo_server_url, auth, + related_product, os): + from redo import retry + import requests + + url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os) + self.info("Requesting {} from tuxedo".format(url)) + + def get_tuxedo_page(): + r = requests.get(url, auth=auth, + verify=False, timeout=60) + r.raise_for_status() + return r.content + + def calculateUptake(page): + doc = xml.dom.minidom.parseString(page) + uptake_values = [] + + for element in doc.getElementsByTagName('available'): + for node in element.childNodes: + if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \ + node.data.isdigit(): + uptake_values.append(int(node.data)) + if not uptake_values: + uptake_values = [0] + return min(uptake_values) + + page = retry(get_tuxedo_page) + uptake = calculateUptake(page) + self.info("Current uptake for {} is {}".format(related_product, uptake)) + return uptake + + def _get_release_uptake(self, auth): + assert isinstance(self.config["platforms"], (list, tuple)) + + # handle the products first + tuxedo_server_url = self.config["tuxedo_server_url"] + version = self.config["version"] + dl = [] + + for product, info in self.config["products"].iteritems(): + if info.get("check_uptake"): + product_template = info["product-name"] + related_product = product_template % {"version": version} + + enUS_platforms = set(self.config["platforms"]) + paths_platforms = set(info["paths"].keys()) + platforms = enUS_platforms.intersection(paths_platforms) + + for platform in platforms: + bouncer_platform = info["paths"].get(platform).get('bouncer-platform') + dl.append(self._get_product_uptake(tuxedo_server_url, auth, + related_product, bouncer_platform)) + # handle the partials as well + prev_versions = self.config["partial_versions"] + for product, info in self.config["partials"].iteritems(): + if info.get("check_uptake"): + product_template = info["product-name"] + for prev_version in prev_versions: + subs = { + "version": version, + "prev_version": prev_version + } + related_product = product_template % subs + + enUS_platforms = set(self.config["platforms"]) + paths_platforms = set(info["paths"].keys()) + platforms = enUS_platforms.intersection(paths_platforms) + + for platform in platforms: + bouncer_platform = info["paths"].get(platform).get('bouncer-platform') + dl.append(self._get_product_uptake(tuxedo_server_url, auth, + related_product, bouncer_platform)) + return min(dl) + + def monitor_uptake(self): + credentials_file = os.path.join(os.getcwd(), + self.config["credentials_file"]) + credentials = {} + execfile(credentials_file, credentials) + auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword']) + self.info("Starting the loop to determine the uptake monitoring ...") + + start_time = datetime.datetime.now() + while True: + delta = (datetime.datetime.now() - start_time).seconds + if delta > self.config["poll_timeout"]: + self.error("Uptake monitoring sadly timed-out") + raise Exception("Time-out during uptake monitoring") + + uptake = self._get_release_uptake(auth) + self.info("Current uptake value to check is {}".format(uptake)) + + if uptake >= self.config["min_uptake"]: + self.info("Uptake monitoring is complete!") + break + else: + self.info("Mirrors not yet updated, sleeping for a bit ...") + time.sleep(self.config["poll_interval"]) + + +if __name__ == '__main__': + myScript = UptakeMonitoring() + myScript.run_and_exit() |