diff options
Diffstat (limited to 'testing/docker/funsize-balrog-submitter')
8 files changed, 311 insertions, 0 deletions
diff --git a/testing/docker/funsize-balrog-submitter/Dockerfile b/testing/docker/funsize-balrog-submitter/Dockerfile new file mode 100644 index 000000000..282f98b2c --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/Dockerfile @@ -0,0 +1,35 @@ +FROM ubuntu:vivid +MAINTAINER Rail Aliiev <rail@mozilla.com> + +# Required software +ENV DEBIAN_FRONTEND noninteractive +# Ubuntu Vivid has been moved to the old-releases repo +RUN sed -i -e 's/archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list +# Chain apt-get commands with apt-get clean in a single docker RUN +# to make sure that files are removed within a single docker layer +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + python mercurial curl python-boto python-setuptools python-cryptography && \ + apt-get clean + +COPY requirements.txt /tmp/ +# python-pip installs a lot of dependencies increasing the size of an image +# drastically. +RUN easy_install pip +RUN pip install -r /tmp/requirements.txt + +RUN hg clone https://hg.mozilla.org/build/tools /home/worker/tools + +RUN useradd -d /home/worker -s /bin/bash -m worker + +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /runme.sh + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker diff --git a/testing/docker/funsize-balrog-submitter/Makefile b/testing/docker/funsize-balrog-submitter/Makefile new file mode 100644 index 000000000..2e46ee493 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/Makefile @@ -0,0 +1,17 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = funsize-balrog-submitter +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff --git a/testing/docker/funsize-balrog-submitter/dep.pubkey b/testing/docker/funsize-balrog-submitter/dep.pubkey new file mode 100644 index 000000000..a1213a57e --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/dep.pubkey @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp +WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ +ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3 +eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe +pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS +Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH +0wIDAQAB +-----END PUBLIC KEY----- diff --git a/testing/docker/funsize-balrog-submitter/nightly.pubkey b/testing/docker/funsize-balrog-submitter/nightly.pubkey new file mode 100644 index 000000000..93c0904d5 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/nightly.pubkey @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC +Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp +SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529 +qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7 +VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX +VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq +hQIDAQAB +-----END PUBLIC KEY----- diff --git a/testing/docker/funsize-balrog-submitter/release.pubkey b/testing/docker/funsize-balrog-submitter/release.pubkey new file mode 100644 index 000000000..20df95946 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/release.pubkey @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9 +NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x +3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs +Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn +qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO +K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN +sQIDAQAB +-----END PUBLIC KEY----- diff --git a/testing/docker/funsize-balrog-submitter/requirements.txt b/testing/docker/funsize-balrog-submitter/requirements.txt new file mode 100644 index 000000000..aec79b364 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/requirements.txt @@ -0,0 +1 @@ +mar==1.2 diff --git a/testing/docker/funsize-balrog-submitter/runme.sh b/testing/docker/funsize-balrog-submitter/runme.sh new file mode 100644 index 000000000..ecf222403 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/runme.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -xe + +test $PARENT_TASK_ARTIFACTS_URL_PREFIX +test $BALROG_API_ROOT +test $SIGNING_CERT + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \ + "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json" + +cat "$ARTIFACTS_DIR/manifest.json" +python /home/worker/bin/funsize-balrog-submitter.py \ + --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \ + --manifest "$ARTIFACTS_DIR/manifest.json" \ + -a "$BALROG_API_ROOT" \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + --verbose \ + $EXTRA_BALROG_SUBMITTER_PARAMS diff --git a/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py b/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py new file mode 100644 index 000000000..17e713069 --- /dev/null +++ b/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +import site +import os +import logging +import argparse +import json +import hashlib +import requests +import tempfile +from boto.s3.connection import S3Connection +from mardor.marfile import MarFile + +site.addsitedir("/home/worker/tools/lib/python") + +from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4 +from util.retry import retry, retriable + +log = logging.getLogger(__name__) + + +def get_hash(content, hash_type="md5"): + h = hashlib.new(hash_type) + h.update(content) + return h.hexdigest() + + +@retriable() +def download(url, dest, mode=None): + log.debug("Downloading %s to %s", url, dest) + r = requests.get(url) + r.raise_for_status() + + bytes_downloaded = 0 + with open(dest, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + bytes_downloaded += len(chunk) + + log.debug('Downloaded %s bytes', bytes_downloaded) + if 'content-length' in r.headers: + log.debug('Content-Length: %s bytes', r.headers['content-length']) + if bytes_downloaded != int(r.headers['content-length']): + raise IOError('Unexpected number of bytes downloaded') + + if mode: + log.debug("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +def verify_signature(mar, signature): + log.info("Checking %s signature", mar) + m = MarFile(mar, signature_versions=[(1, signature)]) + m.verify_signatures() + + +def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key, + mar_url, mar_dest, signing_cert): + conn = S3Connection(aws_access_key_id, aws_secret_access_key) + bucket = conn.get_bucket(bucket_name) + _, dest = tempfile.mkstemp() + log.info("Downloading %s to %s...", mar_url, dest) + download(mar_url, dest) + log.info("Verifying the signature...") + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): + verify_signature(dest, signing_cert) + for name in possible_names(mar_dest, 10): + log.info("Checking if %s already exists", name) + key = bucket.get_key(name) + if not key: + log.info("Uploading to %s...", name) + key = bucket.new_key(name) + # There is a chance for race condition here. To avoid it we check + # the return value with replace=False. It should be not None. + length = key.set_contents_from_filename(dest, replace=False) + if length is None: + log.warn("Name race condition using %s, trying again...", name) + continue + else: + # key.make_public() may lead to race conditions, because + # it doesn't pass version_id, so it may not set permissions + bucket.set_canned_acl(acl_str='public-read', key_name=name, + version_id=key.version_id) + # Use explicit version_id to avoid using "latest" version + return key.generate_url(expires_in=0, query_auth=False, + version_id=key.version_id) + else: + if get_hash(key.get_contents_as_string()) == \ + get_hash(open(dest).read()): + log.info("%s has the same MD5 checksum, not uploading...", + name) + return key.generate_url(expires_in=0, query_auth=False, + version_id=key.version_id) + log.info("%s already exists with different checksum, " + "trying another one...", name) + + raise RuntimeError("Cannot generate a unique name for %s", mar_dest) + + +def possible_names(initial_name, amount): + """Generate names appending counter before extension""" + prefix, ext = os.path.splitext(initial_name) + return [initial_name] + ["{}-{}{}".format(prefix, n, ext) for n in + range(1, amount + 1)] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-url-prefix", required=True, + help="URL prefix for MAR") + parser.add_argument("--manifest", required=True) + parser.add_argument("-a", "--api-root", required=True, + help="Balrog API root") + parser.add_argument("-d", "--dummy", action="store_true", + help="Add '-dummy' suffix to branch name") + parser.add_argument("--signing-cert", required=True) + parser.add_argument("-v", "--verbose", action="store_const", + dest="loglevel", const=logging.DEBUG, + default=logging.INFO) + parser.add_argument("--product", help="Override product name from application.ini") + args = parser.parse_args() + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", + level=args.loglevel) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("boto").setLevel(logging.WARNING) + + balrog_username = os.environ.get("BALROG_USERNAME") + balrog_password = os.environ.get("BALROG_PASSWORD") + if not balrog_username and not balrog_password: + raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " + "variables should be set") + + s3_bucket = os.environ.get("S3_BUCKET") + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not (s3_bucket and aws_access_key_id and aws_secret_access_key): + log.warn("Skipping S3 uploads...") + uploads_enabled = False + else: + uploads_enabled = True + + manifest = json.load(open(args.manifest)) + auth = (balrog_username, balrog_password) + + for e in manifest: + complete_info = [{ + "hash": e["to_hash"], + "size": e["to_size"], + }] + partial_info = [{ + "hash": e["hash"], + "size": e["size"], + }] + + if "previousVersion" in e and "previousBuildNumber" in e: + log.info("Release style balrog submission") + partial_info[0]["previousVersion"] = e["previousVersion"] + partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"] + submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, + dummy=args.dummy) + productName = args.product or e["appName"] + retry(lambda: submitter.run( + platform=e["platform"], productName=productName, + version=e["toVersion"], + build_number=e["toBuildNumber"], + appVersion=e["version"], extVersion=e["version"], + buildID=e["to_buildid"], locale=e["locale"], + hashFunction='sha512', + partialInfo=partial_info, completeInfo=complete_info, + )) + elif "from_buildid" in e and uploads_enabled: + log.info("Nightly style balrog submission") + partial_mar_url = "{}/{}".format(args.artifacts_url_prefix, + e["mar"]) + complete_mar_url = e["to_mar"] + dest_prefix = "{branch}/{buildid}".format( + branch=e["branch"], buildid=e["to_buildid"]) + partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"]) + complete_mar_filename = "{appName}-{branch}-{version}-" \ + "{platform}-{locale}.complete.mar" + complete_mar_filename = complete_mar_filename.format( + appName=e["appName"], branch=e["branch"], + version=e["version"], platform=e["platform"], + locale=e["locale"] + ) + complete_mar_dest = "{}/{}".format(dest_prefix, + complete_mar_filename) + partial_info[0]["url"] = verify_copy_to_s3( + s3_bucket, aws_access_key_id, aws_secret_access_key, + partial_mar_url, partial_mar_dest, args.signing_cert) + complete_info[0]["url"] = verify_copy_to_s3( + s3_bucket, aws_access_key_id, aws_secret_access_key, + complete_mar_url, complete_mar_dest, args.signing_cert) + partial_info[0]["from_buildid"] = e["from_buildid"] + submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth, + dummy=args.dummy) + productName = args.product or e["appName"] + retry(lambda: submitter.run( + platform=e["platform"], buildID=e["to_buildid"], + productName=productName, branch=e["branch"], + appVersion=e["version"], locale=e["locale"], + hashFunction='sha512', extVersion=e["version"], + partialInfo=partial_info, completeInfo=complete_info), + attempts=30, sleeptime=10, max_sleeptime=60, + ) + else: + raise RuntimeError("Cannot determine Balrog submission style") + +if __name__ == '__main__': + main() |