summaryrefslogtreecommitdiffstats
path: root/testing/docker/funsize-balrog-submitter
diff options
context:
space:
mode:
Diffstat (limited to 'testing/docker/funsize-balrog-submitter')
-rw-r--r--testing/docker/funsize-balrog-submitter/Dockerfile35
-rw-r--r--testing/docker/funsize-balrog-submitter/Makefile17
-rw-r--r--testing/docker/funsize-balrog-submitter/dep.pubkey9
-rw-r--r--testing/docker/funsize-balrog-submitter/nightly.pubkey9
-rw-r--r--testing/docker/funsize-balrog-submitter/release.pubkey9
-rw-r--r--testing/docker/funsize-balrog-submitter/requirements.txt1
-rw-r--r--testing/docker/funsize-balrog-submitter/runme.sh22
-rw-r--r--testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py209
8 files changed, 311 insertions, 0 deletions
diff --git a/testing/docker/funsize-balrog-submitter/Dockerfile b/testing/docker/funsize-balrog-submitter/Dockerfile
new file mode 100644
index 000000000..282f98b2c
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/Dockerfile
@@ -0,0 +1,35 @@
+FROM ubuntu:vivid
+MAINTAINER Rail Aliiev <rail@mozilla.com>
+
+# Required software
+ENV DEBIAN_FRONTEND noninteractive
+# Ubuntu Vivid has been moved to the old-releases repo
+RUN sed -i -e 's/archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list
+# Chain apt-get commands with apt-get clean in a single docker RUN
+# to make sure that files are removed within a single docker layer
+RUN apt-get update -q && \
+ apt-get install -yyq --no-install-recommends \
+ python mercurial curl python-boto python-setuptools python-cryptography && \
+ apt-get clean
+
+COPY requirements.txt /tmp/
+# python-pip installs a lot of dependencies increasing the size of an image
+# drastically.
+RUN easy_install pip
+RUN pip install -r /tmp/requirements.txt
+
+RUN hg clone https://hg.mozilla.org/build/tools /home/worker/tools
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+
+RUN mkdir /home/worker/bin
+COPY scripts/* /home/worker/bin/
+RUN mkdir /home/worker/keys
+COPY *.pubkey /home/worker/keys/
+COPY runme.sh /runme.sh
+RUN chmod 755 /home/worker/bin/* /runme.sh
+
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
diff --git a/testing/docker/funsize-balrog-submitter/Makefile b/testing/docker/funsize-balrog-submitter/Makefile
new file mode 100644
index 000000000..2e46ee493
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/Makefile
@@ -0,0 +1,17 @@
+DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set)
+IMAGE_NAME = funsize-balrog-submitter
+FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME)
+
+build:
+ docker build -t $(FULL_IMAGE_NAME) --no-cache --rm .
+
+push:
+ docker push $(FULL_IMAGE_NAME):latest
+
+pull:
+ docker pull $(FULL_IMAGE_NAME):latest
+
+update_pubkeys:
+ curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey
+ curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey
+ curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey
diff --git a/testing/docker/funsize-balrog-submitter/dep.pubkey b/testing/docker/funsize-balrog-submitter/dep.pubkey
new file mode 100644
index 000000000..a1213a57e
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/dep.pubkey
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp
+WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ
+ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3
+eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe
+pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS
+Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH
+0wIDAQAB
+-----END PUBLIC KEY-----
diff --git a/testing/docker/funsize-balrog-submitter/nightly.pubkey b/testing/docker/funsize-balrog-submitter/nightly.pubkey
new file mode 100644
index 000000000..93c0904d5
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/nightly.pubkey
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC
+Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp
+SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529
+qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7
+VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX
+VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq
+hQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/testing/docker/funsize-balrog-submitter/release.pubkey b/testing/docker/funsize-balrog-submitter/release.pubkey
new file mode 100644
index 000000000..20df95946
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/release.pubkey
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9
+NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x
+3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs
+Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn
+qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO
+K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN
+sQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/testing/docker/funsize-balrog-submitter/requirements.txt b/testing/docker/funsize-balrog-submitter/requirements.txt
new file mode 100644
index 000000000..aec79b364
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/requirements.txt
@@ -0,0 +1 @@
+mar==1.2
diff --git a/testing/docker/funsize-balrog-submitter/runme.sh b/testing/docker/funsize-balrog-submitter/runme.sh
new file mode 100644
index 000000000..ecf222403
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/runme.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -xe
+
+test $PARENT_TASK_ARTIFACTS_URL_PREFIX
+test $BALROG_API_ROOT
+test $SIGNING_CERT
+
+ARTIFACTS_DIR="/home/worker/artifacts"
+mkdir -p "$ARTIFACTS_DIR"
+
+curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \
+ "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json"
+
+cat "$ARTIFACTS_DIR/manifest.json"
+python /home/worker/bin/funsize-balrog-submitter.py \
+ --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \
+ --manifest "$ARTIFACTS_DIR/manifest.json" \
+ -a "$BALROG_API_ROOT" \
+ --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \
+ --verbose \
+ $EXTRA_BALROG_SUBMITTER_PARAMS
diff --git a/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py b/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
new file mode 100644
index 000000000..17e713069
--- /dev/null
+++ b/testing/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+import site
+import os
+import logging
+import argparse
+import json
+import hashlib
+import requests
+import tempfile
+from boto.s3.connection import S3Connection
+from mardor.marfile import MarFile
+
+site.addsitedir("/home/worker/tools/lib/python")
+
+from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4
+from util.retry import retry, retriable
+
+log = logging.getLogger(__name__)
+
+
+def get_hash(content, hash_type="md5"):
+ h = hashlib.new(hash_type)
+ h.update(content)
+ return h.hexdigest()
+
+
+@retriable()
+def download(url, dest, mode=None):
+ log.debug("Downloading %s to %s", url, dest)
+ r = requests.get(url)
+ r.raise_for_status()
+
+ bytes_downloaded = 0
+ with open(dest, 'wb') as fd:
+ for chunk in r.iter_content(4096):
+ fd.write(chunk)
+ bytes_downloaded += len(chunk)
+
+ log.debug('Downloaded %s bytes', bytes_downloaded)
+ if 'content-length' in r.headers:
+ log.debug('Content-Length: %s bytes', r.headers['content-length'])
+ if bytes_downloaded != int(r.headers['content-length']):
+ raise IOError('Unexpected number of bytes downloaded')
+
+ if mode:
+ log.debug("chmod %o %s", mode, dest)
+ os.chmod(dest, mode)
+
+
+def verify_signature(mar, signature):
+ log.info("Checking %s signature", mar)
+ m = MarFile(mar, signature_versions=[(1, signature)])
+ m.verify_signatures()
+
+
+def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key,
+ mar_url, mar_dest, signing_cert):
+ conn = S3Connection(aws_access_key_id, aws_secret_access_key)
+ bucket = conn.get_bucket(bucket_name)
+ _, dest = tempfile.mkstemp()
+ log.info("Downloading %s to %s...", mar_url, dest)
+ download(mar_url, dest)
+ log.info("Verifying the signature...")
+ if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
+ verify_signature(dest, signing_cert)
+ for name in possible_names(mar_dest, 10):
+ log.info("Checking if %s already exists", name)
+ key = bucket.get_key(name)
+ if not key:
+ log.info("Uploading to %s...", name)
+ key = bucket.new_key(name)
+ # There is a chance for race condition here. To avoid it we check
+ # the return value with replace=False. It should be not None.
+ length = key.set_contents_from_filename(dest, replace=False)
+ if length is None:
+ log.warn("Name race condition using %s, trying again...", name)
+ continue
+ else:
+ # key.make_public() may lead to race conditions, because
+ # it doesn't pass version_id, so it may not set permissions
+ bucket.set_canned_acl(acl_str='public-read', key_name=name,
+ version_id=key.version_id)
+ # Use explicit version_id to avoid using "latest" version
+ return key.generate_url(expires_in=0, query_auth=False,
+ version_id=key.version_id)
+ else:
+ if get_hash(key.get_contents_as_string()) == \
+ get_hash(open(dest).read()):
+ log.info("%s has the same MD5 checksum, not uploading...",
+ name)
+ return key.generate_url(expires_in=0, query_auth=False,
+ version_id=key.version_id)
+ log.info("%s already exists with different checksum, "
+ "trying another one...", name)
+
+ raise RuntimeError("Cannot generate a unique name for %s", mar_dest)
+
+
+def possible_names(initial_name, amount):
+ """Generate names appending counter before extension"""
+ prefix, ext = os.path.splitext(initial_name)
+ return [initial_name] + ["{}-{}{}".format(prefix, n, ext) for n in
+ range(1, amount + 1)]
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--artifacts-url-prefix", required=True,
+ help="URL prefix for MAR")
+ parser.add_argument("--manifest", required=True)
+ parser.add_argument("-a", "--api-root", required=True,
+ help="Balrog API root")
+ parser.add_argument("-d", "--dummy", action="store_true",
+ help="Add '-dummy' suffix to branch name")
+ parser.add_argument("--signing-cert", required=True)
+ parser.add_argument("-v", "--verbose", action="store_const",
+ dest="loglevel", const=logging.DEBUG,
+ default=logging.INFO)
+ parser.add_argument("--product", help="Override product name from application.ini")
+ args = parser.parse_args()
+ logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
+ level=args.loglevel)
+ logging.getLogger("requests").setLevel(logging.WARNING)
+ logging.getLogger("boto").setLevel(logging.WARNING)
+
+ balrog_username = os.environ.get("BALROG_USERNAME")
+ balrog_password = os.environ.get("BALROG_PASSWORD")
+ if not balrog_username and not balrog_password:
+ raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
+ "variables should be set")
+
+ s3_bucket = os.environ.get("S3_BUCKET")
+ aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
+ aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
+ if not (s3_bucket and aws_access_key_id and aws_secret_access_key):
+ log.warn("Skipping S3 uploads...")
+ uploads_enabled = False
+ else:
+ uploads_enabled = True
+
+ manifest = json.load(open(args.manifest))
+ auth = (balrog_username, balrog_password)
+
+ for e in manifest:
+ complete_info = [{
+ "hash": e["to_hash"],
+ "size": e["to_size"],
+ }]
+ partial_info = [{
+ "hash": e["hash"],
+ "size": e["size"],
+ }]
+
+ if "previousVersion" in e and "previousBuildNumber" in e:
+ log.info("Release style balrog submission")
+ partial_info[0]["previousVersion"] = e["previousVersion"]
+ partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
+ submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
+ dummy=args.dummy)
+ productName = args.product or e["appName"]
+ retry(lambda: submitter.run(
+ platform=e["platform"], productName=productName,
+ version=e["toVersion"],
+ build_number=e["toBuildNumber"],
+ appVersion=e["version"], extVersion=e["version"],
+ buildID=e["to_buildid"], locale=e["locale"],
+ hashFunction='sha512',
+ partialInfo=partial_info, completeInfo=complete_info,
+ ))
+ elif "from_buildid" in e and uploads_enabled:
+ log.info("Nightly style balrog submission")
+ partial_mar_url = "{}/{}".format(args.artifacts_url_prefix,
+ e["mar"])
+ complete_mar_url = e["to_mar"]
+ dest_prefix = "{branch}/{buildid}".format(
+ branch=e["branch"], buildid=e["to_buildid"])
+ partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"])
+ complete_mar_filename = "{appName}-{branch}-{version}-" \
+ "{platform}-{locale}.complete.mar"
+ complete_mar_filename = complete_mar_filename.format(
+ appName=e["appName"], branch=e["branch"],
+ version=e["version"], platform=e["platform"],
+ locale=e["locale"]
+ )
+ complete_mar_dest = "{}/{}".format(dest_prefix,
+ complete_mar_filename)
+ partial_info[0]["url"] = verify_copy_to_s3(
+ s3_bucket, aws_access_key_id, aws_secret_access_key,
+ partial_mar_url, partial_mar_dest, args.signing_cert)
+ complete_info[0]["url"] = verify_copy_to_s3(
+ s3_bucket, aws_access_key_id, aws_secret_access_key,
+ complete_mar_url, complete_mar_dest, args.signing_cert)
+ partial_info[0]["from_buildid"] = e["from_buildid"]
+ submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth,
+ dummy=args.dummy)
+ productName = args.product or e["appName"]
+ retry(lambda: submitter.run(
+ platform=e["platform"], buildID=e["to_buildid"],
+ productName=productName, branch=e["branch"],
+ appVersion=e["version"], locale=e["locale"],
+ hashFunction='sha512', extVersion=e["version"],
+ partialInfo=partial_info, completeInfo=complete_info),
+ attempts=30, sleeptime=10, max_sleeptime=60,
+ )
+ else:
+ raise RuntimeError("Cannot determine Balrog submission style")
+
+if __name__ == '__main__':
+ main()