diff options
Diffstat (limited to 'build/upload.py')
| -rw-r--r-- | build/upload.py | 375 |
1 files changed, 375 insertions, 0 deletions
diff --git a/build/upload.py b/build/upload.py new file mode 100644 index 000000000..c6bc10429 --- /dev/null +++ b/build/upload.py @@ -0,0 +1,375 @@ +#!/usr/bin/python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# When run directly, this script expects the following environment variables +# to be set: +# UPLOAD_HOST : host to upload files to +# UPLOAD_USER : username on that host +# and one of the following: +# UPLOAD_PATH : path on that host to put the files in +# UPLOAD_TO_TEMP : upload files to a new temporary directory +# +# If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out +# the properties file. +# +# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH. +# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no +# properties are written out. +# +# And will use the following optional environment variables if set: +# UPLOAD_SSH_KEY : path to a ssh private key to use +# UPLOAD_PORT : port to use for ssh +# POST_UPLOAD_CMD: a commandline to run on the remote host after uploading. +# UPLOAD_PATH and the full paths of all files uploaded will +# be appended to the commandline. +# +# All files to be uploaded should be passed as commandline arguments to this +# script. The script takes one other parameter, --base-path, which you can use +# to indicate that files should be uploaded including their paths relative +# to the base path. + +import sys, os +import re +import json +import errno +import hashlib +import shutil +from optparse import OptionParser +from subprocess import ( + check_call, + check_output, + STDOUT, + CalledProcessError, +) +import concurrent.futures as futures +import redo + +def OptionalEnvironmentVariable(v): + """Return the value of the environment variable named v, or None + if it's unset (or empty).""" + if v in os.environ and os.environ[v] != "": + return os.environ[v] + return None + +def FixupMsysPath(path): + """MSYS helpfully translates absolute pathnames in environment variables + and commandline arguments into Windows native paths. This sucks if you're + trying to pass an absolute path on a remote server. This function attempts + to un-mangle such paths.""" + if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys': + # sort of awful, find out where our shell is (should be in msys/bin) + # and strip the first part of that path out of the other path + if 'SHELL' in os.environ: + sh = os.environ['SHELL'] + msys = sh[:sh.find('/bin')] + if path.startswith(msys): + path = path[len(msys):] + return path + +def WindowsPathToMsysPath(path): + """Translate a Windows pathname to an MSYS pathname. + Necessary because we call out to ssh/scp, which are MSYS binaries + and expect MSYS paths.""" + # If we're not on Windows, or if we already have an MSYS path (starting + # with '/' instead of 'c:' or something), then just return. + if sys.platform != 'win32' or path.startswith('/'): + return path + (drive, path) = os.path.splitdrive(os.path.abspath(path)) + return "/" + drive[0] + path.replace('\\','/') + +def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key): + """Given optional port and ssh key values, append valid OpenSSH + commandline arguments to the list cmdline if the values are not None.""" + if port is not None: + cmdline.append("-P%d" % port) + if ssh_key is not None: + # Don't interpret ~ paths - ssh can handle that on its own + if not ssh_key.startswith('~'): + ssh_key = WindowsPathToMsysPath(ssh_key) + cmdline.extend(["-o", "IdentityFile=%s" % ssh_key]) + # In case of an issue here we don't want to hang on a password prompt. + cmdline.extend(["-o", "BatchMode=yes"]) + +def DoSSHCommand(command, user, host, port=None, ssh_key=None): + """Execute command on user@host using ssh. Optionally use + port and ssh_key, if provided.""" + cmdline = ["ssh"] + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) + cmdline.extend(["%s@%s" % (user, host), command]) + + with redo.retrying(check_output, sleeptime=10) as f: + try: + output = f(cmdline, stderr=STDOUT).strip() + except CalledProcessError as e: + print "failed ssh command output:" + print '=' * 20 + print e.output + print '=' * 20 + raise + return output + + raise Exception("Command %s returned non-zero exit code" % cmdline) + +def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None, + log=False): + """Upload file to user@host:remote_path using scp. Optionally use + port and ssh_key, if provided.""" + if log: + print 'Uploading %s' % file + cmdline = ["scp"] + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) + cmdline.extend([WindowsPathToMsysPath(file), + "%s@%s:%s" % (user, host, remote_path)]) + with redo.retrying(check_call, sleeptime=10) as f: + f(cmdline) + return + + raise Exception("Command %s returned non-zero exit code" % cmdline) + +def GetBaseRelativePath(path, local_file, base_path): + """Given a remote path to upload to, a full path to a local file, and an + optional full path that is a base path of the local file, construct the + full remote path to place the file in. If base_path is not None, include + the relative path from base_path to file.""" + if base_path is None or not local_file.startswith(base_path): + # Hack to work around OSX uploading the i386 SDK from i386/dist. Both + # the i386 SDK and x86-64 SDK end up in the same directory this way. + if base_path.endswith('/x86_64/dist'): + return GetBaseRelativePath(path, local_file, base_path.replace('/x86_64/', '/i386/')) + return path + dir = os.path.dirname(local_file) + # strip base_path + extra slash and make it unixy + dir = dir[len(base_path)+1:].replace('\\','/') + return path + dir + +def GetFileHashAndSize(filename): + sha512Hash = 'UNKNOWN' + size = 'UNKNOWN' + + try: + # open in binary mode to make sure we get consistent results + # across all platforms + with open(filename, "rb") as f: + shaObj = hashlib.sha512(f.read()) + sha512Hash = shaObj.hexdigest() + + size = os.path.getsize(filename) + except: + raise Exception("Unable to get filesize/hash from file: %s" % filename) + + return (sha512Hash, size) + +def GetMarProperties(filename): + if not os.path.exists(filename): + return {} + (mar_hash, mar_size) = GetFileHashAndSize(filename) + return { + 'completeMarFilename': os.path.basename(filename), + 'completeMarSize': mar_size, + 'completeMarHash': mar_hash, + } + +def GetUrlProperties(output, package): + # let's create a switch case using name-spaces/dict + # rather than a long if/else with duplicate code + property_conditions = [ + # key: property name, value: condition + ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or + m.endswith('crashreporter-symbols-full.zip')), + ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))), + ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m), + ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')), + ('completeMarUrl', lambda m: m.endswith('.complete.mar')), + ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m), + ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')), + ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))), + ('testPackagesUrl', lambda m: m.endswith('test_packages.json')), + ('packageUrl', lambda m: m.endswith(package)), + ] + url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''') + properties = {} + + try: + for line in output.splitlines(): + m = url_re.match(line.strip()) + if m: + m = m.group(1) + for prop, condition in property_conditions: + if condition(m): + properties.update({prop: m}) + break + except IOError as e: + if e.errno != errno.ENOENT: + raise + properties = {prop: 'UNKNOWN' for prop, condition in property_conditions} + return properties + +def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None, package=None): + """Upload each file in the list files to user@host:path. Optionally pass + port and ssh_key to the ssh commands. If base_path is not None, upload + files including their path relative to base_path. If upload_to_temp_dir is + True files will be uploaded to a temporary directory on the remote server. + Generally, you should have a post upload command specified in these cases + that can move them around to their correct location(s). + If post_upload_command is not None, execute that command on the remote host + after uploading all files, passing it the upload path, and the full paths to + all files uploaded. + If verbose is True, print status updates while working.""" + if not host or not user: + return {} + if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir): + print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \ + "defined." + sys.exit(1) + + if upload_to_temp_dir: + path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key) + if not path.endswith("/"): + path += "/" + if base_path is not None: + base_path = os.path.abspath(base_path) + remote_files = [] + properties = {} + + def get_remote_path(p): + return GetBaseRelativePath(path, os.path.abspath(p), base_path) + + try: + # Do a pass to find remote directories so we don't perform excessive + # scp calls. + remote_paths = set() + for file in files: + if not os.path.isfile(file): + raise IOError("File not found: %s" % file) + + remote_paths.add(get_remote_path(file)) + + # If we wanted to, we could reduce the remote paths if they are a parent + # of any entry. + for p in sorted(remote_paths): + DoSSHCommand("mkdir -p " + p, user, host, port=port, ssh_key=ssh_key) + + with futures.ThreadPoolExecutor(4) as e: + fs = [] + # Since we're uploading in parallel, the largest file should take + # the longest to upload. So start it first. + for file in sorted(files, key=os.path.getsize, reverse=True): + remote_path = get_remote_path(file) + fs.append(e.submit(DoSCPFile, file, remote_path, user, host, + port=port, ssh_key=ssh_key, log=verbose)) + remote_files.append(remote_path + '/' + os.path.basename(file)) + + # We need to call result() on the future otherwise exceptions could + # get swallowed. + for f in futures.as_completed(fs): + f.result() + + if post_upload_command is not None: + if verbose: + print "Running post-upload command: " + post_upload_command + file_list = '"' + '" "'.join(remote_files) + '"' + output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key) + # We print since mozharness may parse URLs from the output stream. + print output + properties = GetUrlProperties(output, package) + finally: + if upload_to_temp_dir: + DoSSHCommand("rm -rf %s" % path, user, host, port=port, + ssh_key=ssh_key) + if verbose: + print "Upload complete" + return properties + +def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None): + """Copy each file in the list of files to `path`. The `base_path` argument is treated + as it is by UploadFiles.""" + if not path.endswith("/"): + path += "/" + if base_path is not None: + base_path = os.path.abspath(base_path) + for file in files: + file = os.path.abspath(file) + if not os.path.isfile(file): + raise IOError("File not found: %s" % file) + # first ensure that path exists remotely + target_path = GetBaseRelativePath(path, file, base_path) + if not os.path.exists(target_path): + os.makedirs(target_path) + if verbose: + print "Copying " + file + " to " + target_path + shutil.copy(file, target_path) + +def WriteProperties(files, properties_file, url_properties, package): + properties = url_properties + for file in files: + if file.endswith('.complete.mar'): + properties.update(GetMarProperties(file)) + with open(properties_file, 'w') as outfile: + properties['packageFilename'] = package + properties['uploadFiles'] = [os.path.abspath(f) for f in files] + json.dump(properties, outfile, indent=4) + +if __name__ == '__main__': + host = OptionalEnvironmentVariable('UPLOAD_HOST') + user = OptionalEnvironmentVariable('UPLOAD_USER') + path = OptionalEnvironmentVariable('UPLOAD_PATH') + upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP') + port = OptionalEnvironmentVariable('UPLOAD_PORT') + if port is not None: + port = int(port) + key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY') + post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD') + + if sys.platform == 'win32': + if path is not None: + path = FixupMsysPath(path) + if post_upload_command is not None: + post_upload_command = FixupMsysPath(post_upload_command) + + parser = OptionParser(usage="usage: %prog [options] <files>") + parser.add_option("-b", "--base-path", + action="store", + help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.") + parser.add_option("--properties-file", + action="store", + help="Path to the properties file to store the upload properties.") + parser.add_option("--package", + action="store", + help="Name of the main package.") + (options, args) = parser.parse_args() + if len(args) < 1: + print "You must specify at least one file to upload" + sys.exit(1) + if not options.properties_file: + print "You must specify a --properties-file" + sys.exit(1) + + if host == "localhost": + if upload_to_temp_dir: + print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost" + sys.exit(1) + if post_upload_command: + # POST_UPLOAD_COMMAND is difficult to extract from the mozharness + # scripts, so just ignore it until it's no longer used anywhere + print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost" + + try: + if host == "localhost": + CopyFilesLocally(path, args, base_path=options.base_path, + package=options.package, + verbose=True) + else: + + url_properties = UploadFiles(user, host, path, args, + base_path=options.base_path, port=port, ssh_key=key, + upload_to_temp_dir=upload_to_temp_dir, + post_upload_command=post_upload_command, + package=options.package, verbose=True) + + WriteProperties(args, options.properties_file, url_properties, options.package) + except IOError, (strerror): + print strerror + sys.exit(1) |