diff options
Diffstat (limited to 'testing/mozharness/external_tools')
53 files changed, 8953 insertions, 0 deletions
diff --git a/testing/mozharness/external_tools/__init__.py b/testing/mozharness/external_tools/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/mozharness/external_tools/__init__.py diff --git a/testing/mozharness/external_tools/clobberer.py b/testing/mozharness/external_tools/clobberer.py new file mode 100755 index 000000000..a58b00402 --- /dev/null +++ b/testing/mozharness/external_tools/clobberer.py @@ -0,0 +1,280 @@ +#!/usr/bin/python +# vim:sts=2 sw=2 +import sys +import shutil +import urllib2 +import urllib +import os +import traceback +import time +if os.name == 'nt': + from win32file import RemoveDirectory, DeleteFile, \ + GetFileAttributesW, SetFileAttributesW, \ + FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY + from win32api import FindFiles + +clobber_suffix = '.deleteme' + + +def ts_to_str(ts): + if ts is None: + return None + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + + +def write_file(ts, fn): + assert isinstance(ts, int) + f = open(fn, "w") + f.write(str(ts)) + f.close() + + +def read_file(fn): + if not os.path.exists(fn): + return None + + data = open(fn).read().strip() + try: + return int(data) + except ValueError: + return None + +def rmdirRecursiveWindows(dir): + """Windows-specific version of rmdirRecursive that handles + path lengths longer than MAX_PATH. + """ + + dir = os.path.realpath(dir) + # Make sure directory is writable + SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL) + + for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'): + file_attr = ffrec[0] + name = ffrec[8] + if name == '.' or name == '..': + continue + full_name = os.path.join(dir, name) + + if file_attr & FILE_ATTRIBUTE_DIRECTORY: + rmdirRecursiveWindows(full_name) + else: + SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL) + DeleteFile('\\\\?\\' + full_name) + RemoveDirectory('\\\\?\\' + dir) + +def rmdirRecursive(dir): + """This is a replacement for shutil.rmtree that works better under + windows. Thanks to Bear at the OSAF for the code. + (Borrowed from buildbot.slave.commands)""" + if os.name == 'nt': + rmdirRecursiveWindows(dir) + return + + if not os.path.exists(dir): + # This handles broken links + if os.path.islink(dir): + os.remove(dir) + return + + if os.path.islink(dir): + os.remove(dir) + return + + # Verify the directory is read/write/execute for the current user + os.chmod(dir, 0700) + + for name in os.listdir(dir): + full_name = os.path.join(dir, name) + # on Windows, if we don't have write permission we can't remove + # the file/directory either, so turn that on + if os.name == 'nt': + if not os.access(full_name, os.W_OK): + # I think this is now redundant, but I don't have an NT + # machine to test on, so I'm going to leave it in place + # -warner + os.chmod(full_name, 0600) + + if os.path.isdir(full_name): + rmdirRecursive(full_name) + else: + # Don't try to chmod links + if not os.path.islink(full_name): + os.chmod(full_name, 0700) + os.remove(full_name) + os.rmdir(dir) + + +def do_clobber(dir, dryrun=False, skip=None): + try: + for f in os.listdir(dir): + if skip is not None and f in skip: + print "Skipping", f + continue + clobber_path = f + clobber_suffix + if os.path.isfile(f): + print "Removing", f + if not dryrun: + if os.path.exists(clobber_path): + os.unlink(clobber_path) + # Prevent repeated moving. + if f.endswith(clobber_suffix): + os.unlink(f) + else: + shutil.move(f, clobber_path) + os.unlink(clobber_path) + elif os.path.isdir(f): + print "Removing %s/" % f + if not dryrun: + if os.path.exists(clobber_path): + rmdirRecursive(clobber_path) + # Prevent repeated moving. + if f.endswith(clobber_suffix): + rmdirRecursive(f) + else: + shutil.move(f, clobber_path) + rmdirRecursive(clobber_path) + except: + print "Couldn't clobber properly, bailing out." + sys.exit(1) + + +def getClobberDates(clobberURL, branch, buildername, builddir, slave, master): + params = dict(branch=branch, buildername=buildername, + builddir=builddir, slave=slave, master=master) + url = "%s?%s" % (clobberURL, urllib.urlencode(params)) + print "Checking clobber URL: %s" % url + # The timeout arg was added to urlopen() at Python 2.6 + # Deprecate this test when esr17 reaches EOL + if sys.version_info[:2] < (2, 6): + data = urllib2.urlopen(url).read().strip() + else: + data = urllib2.urlopen(url, timeout=30).read().strip() + + retval = {} + try: + for line in data.split("\n"): + line = line.strip() + if not line: + continue + builddir, builder_time, who = line.split(":") + builder_time = int(builder_time) + retval[builddir] = (builder_time, who) + return retval + except ValueError: + print "Error parsing response from server" + print data + raise + +if __name__ == "__main__": + from optparse import OptionParser + parser = OptionParser( + "%prog [options] clobberURL branch buildername builddir slave master") + parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true", + default=False, help="don't actually delete anything") + parser.add_option("-t", "--periodic", dest="period", type="float", + default=None, help="hours between periodic clobbers") + parser.add_option('-s', '--skip', help='do not delete this file/directory', + action='append', dest='skip', default=['last-clobber']) + parser.add_option('-d', '--dir', help='clobber this directory', + dest='dir', default='.', type='string') + parser.add_option('-v', '--verbose', help='be more verbose', + dest='verbose', action='store_true', default=False) + + options, args = parser.parse_args() + if len(args) != 6: + parser.error("Incorrect number of arguments") + + if options.period: + periodicClobberTime = options.period * 3600 + else: + periodicClobberTime = None + + clobberURL, branch, builder, my_builddir, slave, master = args + + try: + server_clobber_dates = getClobberDates( + clobberURL, branch, builder, my_builddir, slave, master) + except: + if options.verbose: + traceback.print_exc() + print "Error contacting server" + sys.exit(1) + + if options.verbose: + print "Server gave us", server_clobber_dates + + now = int(time.time()) + + # Add ourself to the server_clobber_dates if it's not set + # This happens when this slave has never been clobbered + if my_builddir not in server_clobber_dates: + server_clobber_dates[my_builddir] = None, "" + + root_dir = os.path.abspath(options.dir) + + for builddir, (server_clobber_date, who) in server_clobber_dates.items(): + builder_dir = os.path.join(root_dir, builddir) + if not os.path.isdir(builder_dir): + print "%s doesn't exist, skipping" % builder_dir + continue + os.chdir(builder_dir) + + our_clobber_date = read_file("last-clobber") + + clobber = False + clobberType = None + + print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date) + print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date) + + # If we don't have a last clobber date, then this is probably a fresh build. + # We should only do a forced server clobber if we know when our last clobber + # was, and if the server date is more recent than that. + if server_clobber_date is not None and our_clobber_date is not None: + # If the server is giving us a clobber date, compare the server's idea of + # the clobber date to our last clobber date + if server_clobber_date > our_clobber_date: + # If the server's clobber date is greater than our last clobber date, + # then we should clobber. + clobber = True + clobberType = "forced" + # We should also update our clobber date to match the server's + our_clobber_date = server_clobber_date + if who: + print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who) + else: + print "%s:Server is forcing a clobber" % builddir + + if not clobber: + # Disable periodic clobbers for builders that aren't my_builddir + if builddir != my_builddir: + continue + + # Next, check if more than the periodicClobberTime period has passed since + # our last clobber + if our_clobber_date is None: + # We've never been clobbered + # Set our last clobber time to now, so that we'll clobber + # properly after periodicClobberTime + clobberType = "purged" + our_clobber_date = now + write_file(our_clobber_date, "last-clobber") + elif periodicClobberTime and now > our_clobber_date + periodicClobberTime: + # periodicClobberTime has passed since our last clobber + clobber = True + clobberType = "periodic" + # Update our clobber date to now + our_clobber_date = now + print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime) + + if clobber: + # Finally, perform a clobber if we're supposed to + print "%s:Clobbering..." % builddir + do_clobber(builder_dir, options.dryrun, options.skip) + write_file(our_clobber_date, "last-clobber") + + # If this is the build dir for the current job, display the clobber type in TBPL. + # Note in the case of purged clobber, we output the clobber type even though no + # clobber was performed this time. + if clobberType and builddir == my_builddir: + print "TinderboxPrint: %s clobber" % clobberType diff --git a/testing/mozharness/external_tools/count_and_reboot.py b/testing/mozharness/external_tools/count_and_reboot.py new file mode 100755 index 000000000..9e8ae35a6 --- /dev/null +++ b/testing/mozharness/external_tools/count_and_reboot.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Created by Chris AtLee on 2008-11-04 +"""count_and_reboot.py [-n maxcount] -f countfile + +Increments the value in countfile, and reboots the machine once the count +reaches or exceeds maxcount.""" + +import os, sys, time + +if sys.platform in ('darwin', 'linux2'): + def reboot(): + # -S means to accept password from stdin, which we then redirect from + # /dev/null + # This results in sudo not waiting forever for a password. If sudoers + # isn't set up properly, this will fail immediately + os.system("sudo -S reboot < /dev/null") + # After starting the shutdown, we go to sleep since the system can + # take a few minutes to shut everything down and reboot + time.sleep(600) + +elif sys.platform == "win32": + # Windows + def reboot(): + os.system("shutdown -f -r -t 0") + # After starting the shutdown, we go to sleep since the system can + # take a few minutes to shut everything down and reboot + time.sleep(600) + +def increment_count(fname): + try: + current_count = int(open(fname).read()) + except: + current_count = 0 + current_count += 1 + open(fname, "w").write("%i\n" % current_count) + return current_count + +if __name__ == '__main__': + from optparse import OptionParser + + parser = OptionParser(__doc__) + parser.add_option("-n", "--max-count", dest="maxcount", default=10, + help="reboot after <maxcount> runs", type="int") + parser.add_option("-f", "--count-file", dest="countfile", default=None, + help="file to record count in") + parser.add_option("-z", "--zero-count", dest="zero", default=False, + action="store_true", help="zero out the counter before rebooting") + + options, args = parser.parse_args() + + if not options.countfile: + parser.error("countfile is required") + + if increment_count(options.countfile) >= options.maxcount: + if options.zero: + open(options.countfile, "w").write("0\n") + print "************************************************************************************************" + print "*********** END OF RUN - NOW DOING SCHEDULED REBOOT; FOLLOWING ERROR MESSAGE EXPECTED **********" + print "************************************************************************************************" + sys.stdout.flush() + reboot() diff --git a/testing/mozharness/external_tools/detect_repo.py b/testing/mozharness/external_tools/detect_repo.py new file mode 100644 index 000000000..67466a03e --- /dev/null +++ b/testing/mozharness/external_tools/detect_repo.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# Stolen from taskcluster-vcs +# https://github.com/taskcluster/taskcluster-vcs/blob/master/src/vcs/detect_remote.js + +from urllib2 import Request, urlopen +from urlparse import urlsplit, urlunsplit +from os.path import exists, join + +def first(seq): + return next(iter(filter(lambda x: x, seq)), '') + +def all_first(*sequences): + return map(lambda x: first(x), sequences) + +# http://codereview.stackexchange.com/questions/13027/joining-url-path-components-intelligently +# I wonder why this is not a builtin feature in Python +def urljoin(*parts): + schemes, netlocs, paths, queries, fragments = zip(*(urlsplit(part) for part in parts)) + scheme, netloc, query, fragment = all_first(schemes, netlocs, queries, fragments) + path = '/'.join(p.strip('/') for p in paths if p) + return urlunsplit((scheme, netloc, path, query, fragment)) + +def _detect_remote(url, content): + try: + response = urlopen(url) + except Exception: + return False + + if response.getcode() != 200: + return False + + content_type = response.headers.get('content-type', '') + return True if content in content_type else False + +def detect_git(url): + location = urljoin(url, '/info/refs?service=git-upload-pack') + req = Request(location, headers={'User-Agent':'git/2.0.1'}) + return _detect_remote(req, 'x-git') + +def detect_hg(url): + location = urljoin(url, '?cmd=lookup&key=0') + return _detect_remote(location, 'mercurial') + +def detect_local(url): + if exists(join(url, '.git')): + return 'git' + + if exists(join(url, '.hg')): + return 'hg' + + return '' + diff --git a/testing/mozharness/external_tools/download_file.py b/testing/mozharness/external_tools/download_file.py new file mode 100755 index 000000000..91b0a4668 --- /dev/null +++ b/testing/mozharness/external_tools/download_file.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" Helper script for download_file() + +We lose some mozharness functionality by splitting this out, but we gain output_timeout. +""" + +import os +import socket +import sys +import urllib2 +import urlparse + + +def download_file(url, file_name): + try: + f_length = None + f = urllib2.urlopen(url, timeout=30) + if f.info().get('content-length') is not None: + f_length = int(f.info()['content-length']) + got_length = 0 + local_file = open(file_name, 'wb') + while True: + block = f.read(1024 ** 2) + if not block: + if f_length is not None and got_length != f_length: + raise urllib2.URLError("Download incomplete; content-length was %d, but only received %d" % (f_length, got_length)) + break + local_file.write(block) + if f_length is not None: + got_length += len(block) + local_file.close() + print "%s downloaded to %s" % (url, file_name) + except urllib2.HTTPError, e: + print "Warning: Server returned status %s %s for %s" % (str(e.code), str(e), url) + raise + except urllib2.URLError, e: + print "URL Error: %s" % url + remote_host = urlparse.urlsplit(url)[1] + if remote_host: + os.system("nslookup %s" % remote_host) + raise + except socket.timeout, e: + print "Timed out accessing %s: %s" % (url, str(e)) + raise + except socket.error, e: + print "Socket error when accessing %s: %s" % (url, str(e)) + raise + +if __name__ == '__main__': + if len(sys.argv) != 3: + if len(sys.argv) != 2: + print "Usage: download_file.py URL [FILENAME]" + sys.exit(-1) + parts = urlparse.urlparse(sys.argv[1]) + file_name = parts[2].split('/')[-1] + else: + file_name = sys.argv[2] + if os.path.exists(file_name): + print "%s exists; removing" % file_name + os.remove(file_name) + if os.path.exists(file_name): + print "%s still exists; exiting" + sys.exit(-1) + download_file(sys.argv[1], file_name) diff --git a/testing/mozharness/external_tools/extract_and_run_command.py b/testing/mozharness/external_tools/extract_and_run_command.py new file mode 100644 index 000000000..ab48ee1df --- /dev/null +++ b/testing/mozharness/external_tools/extract_and_run_command.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python +"""\ +Usage: extract_and_run_command.py [-j N] [command to run] -- [files and/or directories] + -j is the number of workers to start, defaulting to 1. + [command to run] must be a command that can accept one or many files + to process as arguments. + +WARNING: This script does NOT respond to SIGINT. You must use SIGQUIT or SIGKILL to + terminate it early. + """ + +### The canonical location for this file is +### https://hg.mozilla.org/build/tools/file/default/stage/extract_and_run_command.py +### +### Please update the copy in puppet to deploy new changes to +### stage.mozilla.org, see +# https://wiki.mozilla.org/ReleaseEngineering/How_To/Modify_scripts_on_stage + +import logging +import os +from os import path +import sys +from Queue import Queue +import shutil +import subprocess +import tempfile +from threading import Thread +import time + +logging.basicConfig( + stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +try: + # the future - https://github.com/mozilla/build-mar via a venv + from mardor.marfile import BZ2MarFile +except: + # the past - http://hg.mozilla.org/build/tools/file/default/buildfarm/utils/mar.py + sys.path.append( + path.join(path.dirname(path.realpath(__file__)), "../buildfarm/utils")) + from mar import BZ2MarFile + +SEVENZIP = "7za" + + +def extractMar(filename, tempdir): + m = BZ2MarFile(filename) + m.extractall(path=tempdir) + + +def extractExe(filename, tempdir): + try: + # We don't actually care about output, put we redirect to a tempfile + # to avoid deadlocking in wait() when stdout=PIPE + fd = tempfile.TemporaryFile() + proc = subprocess.Popen([SEVENZIP, 'x', '-o%s' % tempdir, filename], + stdout=fd, stderr=subprocess.STDOUT) + proc.wait() + except subprocess.CalledProcessError: + # Not all EXEs are 7-zip files, so we have to ignore extraction errors + pass + +# The keys here are matched against the last 3 characters of filenames. +# The values are callables that accept two string arguments. +EXTRACTORS = { + '.mar': extractMar, + '.exe': extractExe, +} + + +def find_files(d): + """yields all of the files in `d'""" + for root, dirs, files in os.walk(d): + for f in files: + yield path.abspath(path.join(root, f)) + + +def rchmod(d, mode=0755): + """chmods everything in `d' to `mode', including `d' itself""" + os.chmod(d, mode) + for root, dirs, files in os.walk(d): + for item in dirs: + os.chmod(path.join(root, item), mode) + for item in files: + os.chmod(path.join(root, item), mode) + + +def maybe_extract(filename): + """If an extractor is found for `filename', extracts it to a temporary + directory and chmods it. The consumer is responsible for removing + the extracted files, if desired.""" + ext = path.splitext(filename)[1] + if ext not in EXTRACTORS.keys(): + return None + # Append the full filepath to the tempdir + tempdir_root = tempfile.mkdtemp() + tempdir = path.join(tempdir_root, filename.lstrip('/')) + os.makedirs(tempdir) + EXTRACTORS[ext](filename, tempdir) + rchmod(tempdir_root) + return tempdir_root + + +def process(item, command): + def format_time(t): + return time.strftime("%H:%M:%S", time.localtime(t)) + # Buffer output to avoid interleaving of multiple workers' + logs = [] + args = [item] + proc = None + start = time.time() + logs.append("START %s: %s" % (format_time(start), item)) + # If the file was extracted, we need to process all of its files, too. + tempdir = maybe_extract(item) + if tempdir: + for f in find_files(tempdir): + args.append(f) + + try: + fd = tempfile.TemporaryFile() + proc = subprocess.Popen(command + args, stdout=fd) + proc.wait() + if proc.returncode != 0: + raise Exception("returned %s" % proc.returncode) + finally: + if tempdir: + shutil.rmtree(tempdir) + fd.seek(0) + # rstrip() here to avoid an unnecessary newline, if it exists. + logs.append(fd.read().rstrip()) + end = time.time() + elapsed = end - start + logs.append("END %s (%d seconds elapsed): %s\n" % ( + format_time(end), elapsed, item)) + # Now that we've got all of our output, print it. It's important that + # the logging module is used for this, because "print" is not + # thread-safe. + log.info("\n".join(logs)) + + +def worker(command, errors): + item = q.get() + while item != None: + try: + process(item, command) + except: + errors.put(item) + item = q.get() + +if __name__ == '__main__': + # getopt is used in favour of optparse to enable "--" as a separator + # between the command and list of files. optparse doesn't allow that. + from getopt import getopt + options, args = getopt(sys.argv[1:], 'j:h', ['help']) + + concurrency = 1 + for o, a in options: + if o == '-j': + concurrency = int(a) + elif o in ('-h', '--help'): + log.info(__doc__) + sys.exit(0) + + if len(args) < 3 or '--' not in args: + log.error(__doc__) + sys.exit(1) + + command = [] + while args[0] != "--": + command.append(args.pop(0)) + args.pop(0) + + q = Queue() + errors = Queue() + threads = [] + for i in range(concurrency): + t = Thread(target=worker, args=(command, errors)) + t.start() + threads.append(t) + + # find_files is a generator, so work will begin prior to it finding + # all of the files + for arg in args: + if path.isfile(arg): + q.put(arg) + else: + for f in find_files(arg): + q.put(f) + # Because the workers are started before we start populating the q + # they can't use .empty() to determine whether or not their done. + # We also can't use q.join() or j.task_done(), because we need to + # support Python 2.4. We know that find_files won't yield None, + # so we can detect doneness by having workers die when they get None + # as an item. + for i in range(concurrency): + q.put(None) + + for t in threads: + t.join() + + if not errors.empty(): + log.error("Command failed for the following files:") + while not errors.empty(): + log.error(" %s" % errors.get()) + sys.exit(1) diff --git a/testing/mozharness/external_tools/git-ssh-wrapper.sh b/testing/mozharness/external_tools/git-ssh-wrapper.sh new file mode 100755 index 000000000..86ea37088 --- /dev/null +++ b/testing/mozharness/external_tools/git-ssh-wrapper.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# From http://www.reddit.com/r/git/comments/hdn1a/howto_using_the_git_ssh_variable_for_private_keys/ + +# In the example, this was +# if [ -e "$GIT_SSH_KEY" ]; then +# However, that broke on tilde expansion. +# Let's just assume if GIT_SSH_KEY is set, we want to use it. +if [ "x$GIT_SSH_KEY" != "x" ]; then + exec ssh -o IdentityFile="$GIT_SSH_KEY" -o ServerAliveInterval=600 "$@" +else + exec ssh -o ServerAliveInterval=600 "$@" +fi diff --git a/testing/mozharness/external_tools/gittool.py b/testing/mozharness/external_tools/gittool.py new file mode 100755 index 000000000..520aeaf38 --- /dev/null +++ b/testing/mozharness/external_tools/gittool.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +### Compressed module sources ### +module_sources = [('util', 'eJxlkMEKgzAQRO/5isWTQhFaSg8Ff6LnQknM2ixoItmov1+T2FLb3DY7mZkXGkbnAxjJpiclKI+K\nrOSWSAihsQM28sjBk32WXF0FrKe4YZi8hWAwrZMDuC5fJC1wkaQ+K7eIOqpXm1rTEzmU1ZahLuc/\ncwYlGS9nQNs6jfoACwUDQVIf/RdDAXmULYK0Gpo1aXAz6l3sG6VWJ/nIdjHdx45jWTR3W3xVSKTT\n8NuEE9a+DMzomZz9QOencdyDJ7LvH6zEC9SEeBQ=\n'), ('util.file', 'eJzNVk2P2zYQvftXTF0sLC9ctTbaSwAfim2BFCjSIsktCLy0SFnMSqRAUuv1v+8MP0RZ3uTQU3Sw\nJXLmcWbem5GWy+Vb0fbCQD2oykmtLDgNDVO8FVBL/NG4y/zOcrlcyK7XxkGrTyepTulR23Rnm8HJ\nNj01zDatPKZHJ7qeMBe10R08aFXL07/MWDw+Wrxn5+nyAs+BfTqtPAn3N94KUxwOinXicFgvFgsu\naqh01zMjCkLfbnzgu/WbBeCFUcddTK0RaKqcUM6CrsGdtbe1G+iZtYKDVCAkmhlg1rvjhRVQoRah\nLuiK21UlrJXHVKaeucaW8IfGbQYW88E8I4Bi8lmAdQaTiKFKq9UGrAauQWkHg8VKK2iZOREZFBOV\nm7xlDdJKZR1T1ZjhkVkRAGOadPk9rBcFnAxXZrWBj2YQ66+A7b4BtpuC7W7A/BGHsaD7sFAawXiR\nLXZzi93Uwgg3GHUDtZ+5Rp65NKJy2lxQJY5hHsW4gtUc6lq+ZNrhfcB2GDAlTuyfkAmVYbwaCMdv\n9kY/S44qOMuWV8xwjxRgN8SpRH6oPx5bC7XWP98fmXmERFQjHWbI1KX4VJdCcXtGJRUxKrRHXklf\n2pattA5jyMGvP4/0kBoQKROB6i+FMdoUywc9tNxb1FJxuL+zBHhnl3AHRYozg15VGDHHZukvVN3C\nmgrNrdv4pU5zsffkjhV8wGVAK8rZ2/XYRcI8k45xLHQSO4BGBrYFONmh9GU9YqHQvFZSecJoKG9O\nHzNPjjn1iQttzFxmFqhpN7EIudqGbe3QFXVOKqkCCf/w9veftn5K+Wkwmw6+rx/rxw0VuREvRGHH\n3Eg3kh0HXEnHJMn3Y9NQwxxXYfncEBrVI6d3bHX1RE3Rh474bbuDe9j+svs1JxgV4U2zp/dGn6dx\npSmHnjMnCm95zXyJwXN5wh4vxrqwWhwG1Ur15JubxmkuUdiAtAHypLRRxLoXok3d5CvEceSplQPx\ngqpOxXHm8maaA4qeJmQpLel+duI4crBFjNbOa9iGMW5jy5xZmyPdoCB7rs9qqtc5km82D3G7n4mK\ncX3RUhXh7Hr9qvlVxfpbG0QyHSVHKHlbtFZcnz+phi+Z/Vo5IuqcJW8jXirRO/jnw59EyAYmZ/wI\nfxFdApbvNA6vqonvcZMnw3JKjaDpojTN3N11AEE/30jFMGnFVFGz5kbFZVGRQXvxXT7OFDTAVx8J\ni/mvA20YDmWJPWg6wSXqOcyWBoe2ofTpo4PwonOSW81REl3vxbofvzPK7snSPc3Zfao53pNZ4YNb\nvzaZ9PFL4RvYJ+FbeENE1Dy0NZ61OuPijXOeQDGWYEHK8NQVcTlWJhau1YzTew6/euZKCKuY0ey7\nqJjMTFoN4+NT8v68hh/2kB8zaXEivNNKTCdEQInx4FdWCif84atP+G9DrEIf/tGODW0iN8eB8/AQ\njYv4v/YMTvYDRjHDXN8EGV0wnBvbaewxlJvgD6ii7yUBCuV/5XDUuv1ekqBYBLt1eS2R/wBE3uXX\n'), ('util.commands', 'eJzdWW1v2zgS/u5fwXPQs9x1laDFvSBA9pDdJnfBtkkucS9XtIEgS+OYG4n0kVRc76+/GZKSKPkl\n2T3slzPQOhLJ4bw888yQHg6H55XIDJdCs7lUTFVCcPHAMlmWqcj1cDgc8HIplWG6mi2VzEDr+o1s\n/jK8hPrvZZEaFFXWz4V8eECRA/xmJ/VT/ADmA/4JKkoSkZaQJOPBwKj18YDhxy9dcfHu7ZwXsPEy\nXXL77vrz3cXlu7coeKoqGMC3DJaGXdiZZ0pJddybdp4WGgaDQQ5z0iXJyjzCfxP2+vXjKlUPeuxW\nHLBslTOumV5CxtOCccHMIsXtgaXFKl1rtkqFYRwNVlwYQBHwBILxOb4baSak8YLg27LgGTfFmmUL\nqUHY92431Mj9EWdyuY7GztA5G+HuI5JB+7oZTq926Rc75x4lSE3uxCe/Hu2KuZjLaOjDeMxup6c3\n0+HO4Vd6yF4FEY4Lrs1b9EvBBZB/xm4pQeQR1hP2lBYVtLrF3IDCf6WOxq2eWzeym02cFG1UZCWh\neBeSEtQDJCCeIvznRQlY0RtnKP7BlRShu/x4XC3z1IBdaN8rMJUS9bDfAAG+M+YI9ptKMBxiUcrI\nBUzOGU6oShBGj2PGblKuIXTUj2lRQH7tniziMHxWmllAnUYIAW4QMNwsMKbizS+gJAq7mHcmOX0R\ncVVGwuZVUawnoSVHMaWj9+wWKzze7oA5V6B0BHA6x9jUecdmkKUVmoAwzqUYGYdiNIJMJW24WNhQ\n5jV60fNPqdKsrHKCwwMKtxNlZZaVaQCL80b7wErjBNY2wp0Rp3xDAPYBZxOxxPSfj/UOWDldjoft\nJO+yIFLZArLHJENTt7nNM8feyG5B9qhR4ezm5upmNCFBCQ2dECEF+hBwXA5xgZIDO6FIlxryrrXs\nDTP7LD67fM+iV/Hbua7Xj78KzKv6IYD7fyoOZifoc3gSiDTKriWICFPMv5mw0WrUyaQ9ztQmxxic\nNEvxGZRqn1tnt3opKKWBVjEN6gnUhCE8FZWEk0spAF/rxU+wbh9ORvjfaNI4J/j0TEOpyVLBnH9D\n677gqvsarfUWbRDauTF8MyDy6MvoTTFqtblvuNkp9MxSjkvRl8vULPDtEmNGgiK3duyFBSvT5ZJW\nOh80W3HNhTapyMC5aJZqQNLELBx39if78Os+jFbAdLUXvmM95Hc4MVli4sucZ8lS1nHFedQPJFTh\nFFL1ybujowmj8fbVUfz2T1vD4T+1DELLLM0efSh/JfkSt6QBBBlRpoUhI27FxFgWQI2MlVabQpn2\nYtrepGwr67fQdkvZg20uYHPfdaFwzL0ZSrMKub1I+hxdLFdEt40LvIYOOW5z7DPgG2SVFWXSR9DI\nFQK7KpooNqLXYgZBpUxCVNNQBoYV3VHH4v+6zDxbQcgTKCQAzLVlxy2OaD25pVwVqUbmtSA9CWYO\nHCgW2NnavrU1Q9G2tGdsc3A8aEbQeBzktrFklEHHnZQjk3KYVQ/R0KPaQxBBZRsulY07C5y8kxN2\ndLyRu7sqUmBBf8lvKVF9GXXOdAYA+/VNDdXzCR2pbEJ0EvhQyNWOngK9QYNvwoh9vyd/6HOACmsw\n4RIjWfokeY6nhrQs7UHKZ3w3WCEscN+ewbXznUY7nI4a91ll000BKshBpNBOKqLGPHqlx3gS2EPm\nUX/9JFBwvBnTTkcXfvpyop2UtCnUN2tn9otU37oDGQ8WCdZ4a6zFTY61w8vAxRPGH4SkmhrH8XBf\nIfNbb2vv7NBWpJIW3lbUoykuNWljQiNvU2Aa4k7FcK8Swz4sMcvy8TNrJvWeWyDwzNJbCgw5zRBE\nmuDgA+U2HRyjvkbPefH5T4CG/1lWTTgBE1gO0AXAMuo0M3VLhOfpxJUEx/lcZEWVQ+L7WnuLMKHS\nZhIMcP38a1uatn0ISp3rMLobuvKHPQaYurduOgc/M3c3FLUU7D7xQa2IJrlpJmvcGFmqPaASbSps\nI7xQbC4hLWPnqDsXVXfvsZYV0wtZFTmVc6rttuw3jQxSX5Yu0RbANq1AI/G7lJUgm600pxeLvsfx\nOaxwuaw0eWC2NqDHk0bNHNK8kNljc9rlfXeEfYxVu1Oqb6fvrz5N3amuk5LNZCqfg+c6nN/nUOu9\ncMKGbdbtOuju7UL8iSscvLg+a05e7uv53OnaXO+KjMVNoEmjtR10W8eIlLxbQu2oA3Qmc2B/2Ogu\nXlK3e1J8EQ+2oQ6oTr3NLujZq4HORDe8cW8QdJ0vuRlAUmwVOWAfsRPHBQpc6njvufxl0qVpU7za\ne4C4cXOwfeu13+X6YP/tAZ7QnyChQ2xE/7W8NqXcp64f5yyLNANiNHs9qBdYZIpYlcgk3v6VVI8a\n2cfQCaESCEx/rhK5XOmYTbHk4QRkkB8gVVhnrIOubk/PrUR32MrBHaWiHyR6fIUGz5Us2aziRT6T\nBsk8fYK4vrceB0eYugO6IWuIz2w/bO0Z1JmecJ14fbbfYH7StDJxZtVTGXUMLXZ6o85lPWQ1OxKI\n2wsCrA06dLHDkfUyOicv8GA3U/IRz3TYxD3qMBtqIVzTUF8IfXCGi+R+jfYLeomQA/YvPNTN1zZk\nOVeQGanWhBPiisMVHfgOXR8CbWgrpQg8dD8y8Dtli1LmdqMJO/rL0ZEPFC2huxiiZOkuqXGXvqZ0\nAre/KbgbY2vTz5ILL49GxoGTMR/vXMAmtqmuT6wLxBOzKtNtQsm1tud1qpk07JwRyLGndjzRHbaG\nA6cajJwsmS/yxAaiFz2n6gkbCTPqBq6FSWrvFqLGNHu5dJdc/TTe7DgP2AXVZvHoKrQ9Mq5Q3xxT\nD0/hE8wZg1MCK7EdvpxukVOmGcoBykws0aS6teViVLIHaTsDyQogCdz+UGGZYIucN9Qf+uj2gOki\nHdh19Ocm3Bu4pGA3U3uWh1zVzglYst+cH7D31gNYnm3zQor0sqsbgzA5dmmx0yoL4t4sn089bWmg\nbGCNTHwQspPtGfs0RDc/AudZRizlLwtyt9aOxLdQm15rAyWVc/9bXezetL8/+RkY02joswM5c/iR\nZ0pqOTfDwG5fMu0PcJ3lsW3iNd1p4dHn89/vLi6fWbczG8K53qxtZNvUpzql39if7+Y8Y2FBqimV\n1iCAxYNZ6PD8xT6e/ju5Pp3+I24UuJb2DGQ9nBVyNgMFKl6u486FWaqRxEzX5e5CiXZq6QjpsGir\nquM2QoGfNvqKn799/Tpi39mVe2pGs2zDseEi//vncZhWXVRv4dHA7/Vd8iiHgh2es8N/siFW0RGe\n/brVYDPN+hIsttnh7XYZYe/UKSBExOnM/xLc/C4c34I5x+9TYxRHWgN9F/WdNwmmn198OEtOp9Ob\nix8+Tc+Sy6ubj6cf6p1v8ZABjuDxFOLwgp2UvZJNLbUT+5VAHZbeFhLnxf7+m4hv9XkPBRggCzaX\ntSVvPkdHUC7WP33H5wguWqU3luEXvnodvx6FFRGnJin6CLFlhX05um8vxVyldO//et+BSJ2L8YjV\npdc+xr1ClWE3zkXVcv+LanC4VaviH3fH6/3FzdmP06ubz93d+1TwIvp/MYYCFn8RkDY32BHlnprt\nfNuowvsa/lug8V+mJBic\n'), ('util.retry', 'eJytVk2P2zYQvetXDFwsLDuC4C2wORhxsUHQFgWKnHqXaYmyiUqkQ1LxGkX/e2dIivpy0h6qw1oa\nDh9nHt/MjmivSluwouVJrVULdSdLq1RjQPilm2ZX49dKJS1/s4049YvB0jLJzlwnwdqo81nIc4K/\ncOi/8jO3v+Mr12lRSNbyotgkSVLxGjS3+p6y0golM2DW8vZqzeElA9NwfqXgDu93GbTsrRgsL7AF\ntCYQH4dT8LeSPJQ0h/Tn/j3bZFA2nMnuevisJMdj9Bkd0Pznzb3+9fdm77BWq9Un1jRw9AGtgdHB\nou1aUDVaQ3hrR5qBTlrRgLBgurLkvDJDRJgb6xqLyYNV8JLDMUa/BmHAXjjIrj1xTciGI5uVIdcb\nEzainLi9cS4jL9kM9/0OmKygUt2pIRNn5cVT0W/J0C3CTbOZULrOAY5zEl2kDGx3bThuiTiRWsqD\nYfoX1TUVRgsl684Xm8NvNQwwoDBbTa4S/yjDI1AjjOUVCPnobKY5aCYMOjgJ9peSEXl3uAm8qNOA\nFVxF2/JKMMubuwvjGK7e5XLV6quo0ItYK/Gm2QkzwwsksBHrbm0KBqy2mASmELMnxD7hz4pU1bVc\nWhOBQohwZYZCwwsTnpu76nSvSV92BKf5l05o1NUSCUPEwzTKBCOSlIEjHnFckbp1ScH1WxtuTETO\nI86R9L526R+9+D3P/SU7NYnSkkBiFBQ4pQBY8YOY0HjsKVxj4bgFSpR6Q7CHwt6M16SyMXWlB9dg\n876inlY8fBj6wX6QjzrnFT9153Q19X6qwBHgJDc2r+AJ0lHbgOkxo66z8YFI7GLP7u12EUiQhA+H\nWI5DJKjd/QSWQhOyVunKCXsP1FeoRJ8MysJeXA/a41ffhPz7agISn1U4EX4IKfQN01id0u6Nf/VQ\n+CFD+LE4uO00qsNtS7fklcF2G/yjqy+/RTNdphZYj7lREQwVv4dVRl8FMXD4Q3d8Gg3ebrjt/SLf\nsJAuduBNPGL+m4T/Kr4S36QyidwSbWM1Ttih1jE/b5DNT7D7D+f9wlAfVVCQu+kq9vUTrxV1M/LE\nJYzl8T3TMyhw4UPW3K2n3/EaAj+M3rfw48JzluWkFJYZz7En7hNvGg2E7AZjLSTKf1YiEt5RbQ1z\ngHB9YOvV10vUfwWheoD1eg0f8T9hqTSz2EKQ2zBHbHLszqylTtYZHEu8/+sA7tmiA2ulRhrL8zyZ\n+8Zh5Hm3G48jz7sB5cR0utlPYEKESfQpImRRowIVxkmNebTt1Q1a3jqeIMZbyeWKA9S8dveP6tyz\nQXhh2PGbwrjjfxBjxPS39Ti7gmR21DLE5PFqyB3v+3U2OsY5EEsjBP3vIlhwFlEKYb/D0v/M0CN2\n7oLjNNTHkvwDPQB6iA==\n'), ('util.git', 'eJzNW+uT27YR/66/ApF7IymWeEk/Xuam4/iReJrGntiZdMZ2JEoEJcQUIRPgyddM/vfuAyDAh+S7\nNkmrGVsiCSx2F7u/fRA3Ho+f1eXGKl0aketKqNLKKoUb5VYcld2J3XY8Ho/U/qArK7Txv0y9PlR6\nI01zp66KQ1oZGV0Xau2vKjka5ZXei9qqItno/T4tMyP807pcbvbZHIbt9Y1cHlK7m9PdD7WSFp9F\ns3NVSD/TpLlc1mWhyvcjv1aht1vgfwTf4tpfJVtpv4Ofspoul2W6l8vlbDQabYrUGPFE5mld2Fe7\ntJJfp0ZejQR8DvBo1H0EFLu3pkgok7lY7tP3cpmujS5qK6eVPOgZk1K5wKvE2LSyBhU7HaMYV5eX\nYzcEPw/EP4CCcE9QhUZ4cs0gVA5wgfTeFLKMCb1rBuFTGOSfXZixuIDtS3ByAiTxe4r/zWiKLIDD\nMRIRpbZgBUTgqkuuS4AkHPEAW1c8yykD9L3ES1J2rIu1sgZoeXtJUMpDoWxEbaeN5SFgQsmHWoM2\ncVpSSlvozVyMx7NRpIv+QGKzMLZSh+kYVBOmOE69KL9oVU5xvblgdTD3u9QA9zfKgGdMM4mP/aUT\nA9ziByJlxOuqlrzFPELIj8qAkKBGnIoOhDNsdRtpNDbu6ZvJVtnJXEzAWvFrsdAl7Ekp6aL8chKW\nfzcXm2N2jYRn0f6QUMgI7+fHjTzEXpo8TotCZi/56mlV6eqqO/tZWoD7xvLnjeg57uI5yWlAR/DE\nKZyfbdJSrKVIxbpKy81OANrYdCvwWXIfFZmdPi6AKKkmmzTc/TmKUSVYKmtlDf5/Tc+CYp7DY5UW\n6l8SPBcMYX+wt+QVRlld3YrUsmbE85x+eI0BGgplyonlKXOhLOBvUaDGGBQz1ibMW+HCKxhOYs2F\n3ckS1Qp32VH9xE0lUwsTvXZho9C7vekrk6mKZIkgCAwwUWWup2NaFuMAgMdctNUawe40PJGFh078\nYDhBfeF6BQg5sBgNi3CFnJGVm89ao06x1RkGEralyzur8a42QWbamd+WYEhamEDPH4hv/BbloOb3\nQtcWl4ebADqw+1Y7/XNM3ctM4QUwJTdgCjgENORoscxoBLSZ8N8tW0YifmLP2SHhHez5EQccagA8\n0AFodw+hSB0K3nrj6MF9AFe07AIZMRiqMjYOFBu424ElbnRpUxiK4VjTDFnamENH7TtpJ8ZLA0SR\nv7YgqjK278CwFRgRYaSJrYRd8MUrcra5iBQO+pOJrKoSgs21+OsX7a14IL4H602blUFFSCFJEgBL\noXNii4UweEn+xU6Vdgg1JFr3q1ShnztO0J8CAwBBYKgNCCEMMFDjMPr1YcJe8m7AF07NDnNGbSsX\nY3YGmDhzcauFhnjfI5JZAlmKtbF/DaC0Uwio8AYgKhMwjWziPvjQhsTeliOqgqQRvr7UB0hS3oxh\nMfBXcN+bBcV9vFgs4O4CVhlH4D0XgBXgTdcxkecvn85iM8EHyTEFLJ6Jz65Fx1JaTDbWWNtDjWkF\nzeU1ErDpbDpLOFEIK6BCga0Imkpd7QkxBrCKKc9aUQc0DLOnDaFr1j5gYnRrgNY4QUXNehGMSf4+\nMQxTM8fFCYthT4LcCsADf6OlBLdDZOco9gx+NXHHMEAphg02Nmtkkc9pRiW3dZFW7aE07JJkdkYI\nSbesbN+qRwN+BACWK5cwrbUu+BeIxw8rmZB3skeeMk0qPO5mfJHVscOYJUn/SZtSeRiLWTluxjjs\nUTYcA50tDOAJTsAxscY8Ac4oplkr3c3c1hvYeooGlG3POTK4/U8LiFMlYLzpshMbDGXpoF69/gXM\nwTCc5Rq/A4EJL07Ul27kOaLMRkTVRVkqQWmXAm0YdZzMQGqRR8lGcqwUJP/jC/O2xFqntbSHyk0h\n0zKuRR6I10cNNpNDfNvDMyPGNAatZK+zupCYZBx3CvJVir0QNY9SHFOIk0aLPK2SBpxbSSpRIXPM\no/+zicM5p/wTpsbMplm2xFTF+r3iC6qnmotIFnCgR1mG6M7PKLPOxCqatvL+DEUU4JPHf0wXVvhj\nxVYOu0MNABi8itZZeRftScuDyAQyzsiHOY2kn0UG6UZAFXdnSV9JyygFkwhdvNR34BGWXMC0+/G5\nbfjs8ziMn54zxs8bWbopcwwC32PKojhlcduVaYm5ioN4FerGDugFQRY3d4W28/Y2BG3IORaglEp2\nwA3vm2mUFOypHwHJnt3sphX6oHk4ffvq4Uy8neYSbr6d/QWdEsZIs0kPqMOgvTkt1Arv+8F4vk+2\nla4P0y/7xnM/wznvIIM2j6lZJtf1FiHmCs2BXISHIkiE7sX+1jEFWjlrNj40RBOuY667QXzUnwCg\nhCkbmtNQDYesmharUDahjPD/9AgQemFmjvfTypuH9aIK8F5+OxDC2kwCbrR5vDCf5Cswc3eo9N7s\n2k1z0WpwXKMeQ6vFXdaHDOLOEkdeU8UdlOBbgNfdniDoTGEeZhwNigdMotMxwI6fAdeF1ICKshUO\noup+B/uz8rysEDVWjs+V2OzkBiorqjqxM0rUGMMTNpMnmsMV1o20BOw6VmO8yi49AEDMwbs3RU2q\nh6TMqHVxC6zq9VpW2EGlVIMaOU3vwYlFDIINzLkEttjagOq1NpIgzY0Sawk4IhvGnMiNHTf6Q2rD\nTdiWmjmFkOWNqnSJHd3p+Jvnr5evvn30w9Pl149ePV0+ef4D2A3qfDa8St9bmiZl466tpmWbi05V\nQImMCZvezB2y+JgAstBmkB5EDJI+qRkbZcLNyMGODVXouJehFURuFGY1k1pFG7GBfa1moGtuobW3\nGyQgeG0V6CYaytr2I1x18pS+wHDbyyCzx7QqgUvgV9dFhuW5ay3EbYoL8xVUHCZdU58Dn8B3LMsc\nV1qi4ANsxhZDqu497O0D1Sv9FjfXHp3q/DF6H/JFkzr9MVdFnyjL3Yhust7vi7U0BYDo0gOBjgtV\nFHgzNVNDJd/UZ19FLtzr3LHFhwZYJN85a+x2YkKf06UwsGVosAAJgJd0j+j0bazPTqhJXAXWN9d+\nX+6BeAGLVEcFewziUqICOmmKIv+hZ4NY774DUrvvNuAzWvueH72eIazWdcWMopbijJnUobY7Kw5F\nupFnfTx24s37Jb3Y+lSVRIqB2lCVmfyY4Lzx7IxlNYQHzGuooRrGt/coaoEODDmzhU5zEDuOEnJX\n0N4BQg24OVsw6dqpLm0i75wDHMpzlI7CLr1xwat5z5IWmI7eUjfd6HnTPIWaH5UsSknrOAKUiYKV\n3todvhBkr9dLvn0ddYviVzmwW+2deoAFYKbRFYmjwLQwB7lRuZKQdENxiD1azJ7ljax4yVC+h1XD\nmwl8Bdd97dJ648Srx5ylG1unBcRsZCIXbM6wNHDoRMc6iAWPSPhMgAz56PbAO3L+aS7RfD/9gmxI\nWdT1CZtsmi1ym6PsydX9zvj7V4OY1QWJZ0QCnRUkM4wRjeu2xvYiIhN4/eLJiyvxLWAb+CYtzHkq\nYYeByuU9Kc1c2nRrLv8Jnx6R6P1Yz5riD1GP+zIc5jrwNOvNHX5pcXeKPUjsvBO5V7sxaO6V3ksy\ne7CB0oojpGzbzwbGPeZgFSEkBpJKLrgd350QgIu6/2FPaG8hUC7a4W8gmvhPHAfPDQuvBfxn0Fju\nt8/Rfrg3XnjblTHXYw0xRJXj++/23ej+IXseZaLNDpzMQO+5Cffd9n6a0V3sxIj2Zve1Pbj1saOx\n1v8jHzuRNP+P5AcXhmyOsRONh1u6oaHBgk7Yoia+A+JxOkqihmqVH33c51bkRh9uvYquKPn3UeLK\ntwJyX827KBMFGYIahXgcOSAe34HYAhE4NVGUjsNGs0Y7Tf10hCOIagdrp4fLCzOhTlcvFg7owLCD\nIIM+fgO/xkJSgy8wPZHxkNRhS3NXvPYkDENcyhDXO+4Bnp6hnZqeyI6bZkifBZVHfY22oNxpHzyL\nAXQaIxmaHk/1bftTOTw3V9qtFq4iOXHvN29C4+UxUjWhCY5bSim7wZ5J04khu4bbFMgg+8R0jmDB\nv+iifDMR4jWkT0ddUV1I5uyPYdCJjju3ULiYodNu/U4K94NhBC5CY1o9H6TO4nePh6CUUXltGuZq\n8JEwOdIWUXBKJBKQTw+K506ZNM0dt7XnK9wTJSj2NlngIcx4ZC3q0lULkaLcnChaYvua79IZiS7N\nNt3HsUIJbXhC29kGgb9508s2yvM6Vto2wuj3kDN3X/b6j4sQf5e3a51W2XM8U1LVBzvAUi9tult0\nkf7xdAxhfl3IfdvSnDpP6gc/eKJElXVYvh8/g9pfukMs8RaKPIXCMvsKvvnhOoUy0OrQD3aW0n0T\njOp3RyrexW2YwTDk0/ofwYv5BMflYuHkQ2/+WwCjfZZQqzSbThaLUi+oLtW1nQSL9WGrNUl+tDjp\nDb6ZpvNu0UG1TmsyuzqxHD+dBIkbqgEL34XTIc25EEd8UHRnYdzojIKbx9rBYDDYFo967CFdbdCV\n4jtAaQsyXG+b37G4Tja3tV2TOyEYKqVCUPUAiz0lX9kPQxAznTVvN3HlqE2gaSorsa7okJNbHtb7\njvOPXVpuZYDFTJkNuFl0eM61MLpFP8Sbo8Iak9ZOrRv7EyFrM+rnL8SUqxpaFi7XstDHGVW+utpw\n8c0lJfVFHJkMjDGHf+WGMhlEPb3fA5arzPj30nvq7iPAc88EKO35NFrpzj0hHZvC00wYC7pJIFbx\n6Qv5oVaANKgRoD1piOD0xYJnTeYeQJQ/EEY9nAo1vr4VugAuBURFQ6fINb1dGeqj9LteXSf2vuWP\nRvF784bGQzH5+YtJdMg5GH337GcbdxwW9ByVHcLnT5MLc7lPIfuqOINrzPsMmrVnc+437bx96uT7\ndxWaCXuZ7yL0p3y7X6V0Hbzv0Z36cSjh4gHY/+hkWNR8Adv0zkVAfyLfwiMIhA53TpS4O9RLlOgs\nYpwuuQwpfu/UywfukC6cCv+ocVbsYPA/W+/9udG8KRn/D8P5A/FYlzeycraBzeCy+dMHPopGh2sn\nWMpxyRhOVTvjpz9RGPobjKGEgZTR+Bwd+ojThmDTcdbwhDqZbHj4LPQTmSXqAXKnEUq7jWziBebO\n6a1vRTMxKE/1RnHjVUOsoLNOrkFKb8GpGkhxxUNdbSV6CUY2d+TIydTOTpCBySyAbwfvVN7y5k7J\nFoiNH1JL0x1uuPw1nvTb5a+O7m9X7VERfESDxgk41z7F9+29yjLATQsyW4gTX0THIvuW2Od/B3W0\n+aPZnZ0IOL+Doj8/x/HnEad/ih7/O25mztFPhK/4kJWLXPTnOL2TVZzzNClBOJS6wvErn+AVt3R8\nIjom0SRyJ48ohwNW7ogyXnz79NETf2qP/yztPqeoXHw4czr03yOfFDU=\n')] + +### Load the compressed module sources ### +import sys, imp +for name, source in module_sources: + source = source.decode("base64").decode("zlib") + mod = imp.new_module(name) + exec source in mod.__dict__ + sys.modules[name] = mod + +### Original script follows ### +#!/usr/bin/python +"""%prog [-p|--props-file] [-r|--rev revision] [-b|--branch branch] + [-s|--shared-dir shared_dir] repo [dest] + +Tool to do safe operations with git. + +revision/branch on commandline will override those in props-file""" + +# Import snippet to find tools lib +import os +import site +import logging +site.addsitedir(os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../../lib/python")) + +try: + import simplejson as json + assert json +except ImportError: + import json + +from util.git import git + + +if __name__ == '__main__': + from optparse import OptionParser + + parser = OptionParser(__doc__) + parser.set_defaults( + revision=os.environ.get('GIT_REV'), + branch=os.environ.get('GIT_BRANCH', None), + propsfile=os.environ.get('PROPERTIES_FILE'), + loglevel=logging.INFO, + shared_dir=os.environ.get('GIT_SHARE_BASE_DIR'), + mirrors=None, + clean=False, + ) + parser.add_option( + "-r", "--rev", dest="revision", help="which revision to update to") + parser.add_option( + "-b", "--branch", dest="branch", help="which branch to update to") + parser.add_option("-p", "--props-file", dest="propsfile", + help="build json file containing revision information") + parser.add_option("-s", "--shared-dir", dest="shared_dir", + help="clone to a shared directory") + parser.add_option("--mirror", dest="mirrors", action="append", + help="add a mirror to try cloning/pulling from before repo") + parser.add_option("--clean", dest="clean", action="store_true", default=False, + help="run 'git clean' after updating the local repository") + parser.add_option("-v", "--verbose", dest="loglevel", + action="store_const", const=logging.DEBUG) + + options, args = parser.parse_args() + + logging.basicConfig( + level=options.loglevel, format="%(asctime)s %(message)s") + + if len(args) not in (1, 2): + parser.error("Invalid number of arguments") + + repo = args[0] + if len(args) == 2: + dest = args[1] + else: + dest = os.path.basename(repo) + + # Parse propsfile + if options.propsfile: + js = json.load(open(options.propsfile)) + if options.revision is None: + options.revision = js['sourcestamp']['revision'] + if options.branch is None: + options.branch = js['sourcestamp']['branch'] + + got_revision = git(repo, dest, options.branch, options.revision, + shareBase=options.shared_dir, + mirrors=options.mirrors, + clean_dest=options.clean, + ) + + print "Got revision %s" % got_revision diff --git a/testing/mozharness/external_tools/machine-configuration.json b/testing/mozharness/external_tools/machine-configuration.json new file mode 100644 index 000000000..29118c0fd --- /dev/null +++ b/testing/mozharness/external_tools/machine-configuration.json @@ -0,0 +1,12 @@ +{ + "win7": { + "screen_resolution": { + "x": 1280, + "y": 1024 + }, + "mouse_position": { + "x": 1010, + "y": 10 + } + } +} diff --git a/testing/mozharness/external_tools/mouse_and_screen_resolution.py b/testing/mozharness/external_tools/mouse_and_screen_resolution.py new file mode 100755 index 000000000..29e46e1bc --- /dev/null +++ b/testing/mozharness/external_tools/mouse_and_screen_resolution.py @@ -0,0 +1,153 @@ +#! /usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Script name: mouse_and_screen_resolution.py +# Purpose: Sets mouse position and screen resolution for Windows 7 32-bit slaves +# Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com> +# Target: Python 2.5 or newer +# +from optparse import OptionParser +from ctypes import windll, Structure, c_ulong, byref +try: + import json +except: + import simplejson as json +import os +import sys +import urllib2 +import socket +import platform +import time + +default_screen_resolution = {"x": 1024, "y": 768} +default_mouse_position = {"x": 1010, "y": 10} + +def wfetch(url, retries=5): + while True: + try: + return urllib2.urlopen(url, timeout=30).read() + except urllib2.HTTPError, e: + print("Failed to fetch '%s': %s" % (url, str(e))) + except urllib2.URLError, e: + print("Failed to fetch '%s': %s" % (url, str(e))) + except socket.timeout, e: + print("Time out accessing %s: %s" % (url, str(e))) + except socket.error, e: + print("Socket error when accessing %s: %s" % (url, str(e))) + if retries < 0: + raise Exception("Could not fetch url '%s'" % url) + retries -= 1 + print("Retrying") + time.sleep(60) + +def main(): + + if not (platform.version().startswith('6.1.760') and not 'PROGRAMFILES(X86)' in os.environ): + # We only want to run this for Windows 7 32-bit + print "INFO: This script was written to be used with Windows 7 32-bit machines." + return 0 + + parser = OptionParser() + parser.add_option( + "--configuration-url", dest="configuration_url", type="string", + help="Specifies the url of the configuration file.") + parser.add_option( + "--configuration-file", dest="configuration_file", type="string", + help="Specifies the path to the configuration file.") + (options, args) = parser.parse_args() + + if (options.configuration_url == None and + options.configuration_file == None): + print "You must specify --configuration-url or --configuration-file." + return 1 + + if options.configuration_file: + with open(options.configuration_file) as f: + conf_dict = json.load(f) + new_screen_resolution = conf_dict["win7"]["screen_resolution"] + new_mouse_position = conf_dict["win7"]["mouse_position"] + else: + try: + conf_dict = json.loads(wfetch(options.configuration_url)) + new_screen_resolution = conf_dict["win7"]["screen_resolution"] + new_mouse_position = conf_dict["win7"]["mouse_position"] + except urllib2.HTTPError, e: + print "This branch does not seem to have the configuration file %s" % str(e) + print "Let's fail over to 1024x768." + new_screen_resolution = default_screen_resolution + new_mouse_position = default_mouse_position + except urllib2.URLError, e: + print "INFRA-ERROR: We couldn't reach hg.mozilla.org: %s" % str(e) + return 1 + except Exception, e: + print "ERROR: We were not expecting any more exceptions: %s" % str(e) + return 1 + + current_screen_resolution = queryScreenResolution() + print "Screen resolution (current): (%(x)s, %(y)s)" % (current_screen_resolution) + + if current_screen_resolution == new_screen_resolution: + print "No need to change the screen resolution." + else: + print "Changing the screen resolution..." + try: + changeScreenResolution(new_screen_resolution["x"], new_screen_resolution["y"]) + except Exception, e: + print "INFRA-ERROR: We have attempted to change the screen resolution but " + \ + "something went wrong: %s" % str(e) + return 1 + time.sleep(3) # just in case + current_screen_resolution = queryScreenResolution() + print "Screen resolution (new): (%(x)s, %(y)s)" % current_screen_resolution + + print "Mouse position (current): (%(x)s, %(y)s)" % (queryMousePosition()) + setCursorPos(new_mouse_position["x"], new_mouse_position["y"]) + current_mouse_position = queryMousePosition() + print "Mouse position (new): (%(x)s, %(y)s)" % (current_mouse_position) + + if current_screen_resolution != new_screen_resolution or current_mouse_position != new_mouse_position: + print "INFRA-ERROR: The new screen resolution or mouse positions are not what we expected" + return 1 + else: + return 0 + +class POINT(Structure): + _fields_ = [("x", c_ulong), ("y", c_ulong)] + +def queryMousePosition(): + pt = POINT() + windll.user32.GetCursorPos(byref(pt)) + return { "x": pt.x, "y": pt.y} + +def setCursorPos(x, y): + windll.user32.SetCursorPos(x, y) + +def queryScreenResolution(): + return {"x": windll.user32.GetSystemMetrics(0), + "y": windll.user32.GetSystemMetrics(1)} + +def changeScreenResolution(xres = None, yres = None, BitsPerPixel = None): + import struct + + DM_BITSPERPEL = 0x00040000 + DM_PELSWIDTH = 0x00080000 + DM_PELSHEIGHT = 0x00100000 + CDS_FULLSCREEN = 0x00000004 + SIZEOF_DEVMODE = 148 + + DevModeData = struct.calcsize("32BHH") * '\x00' + DevModeData += struct.pack("H", SIZEOF_DEVMODE) + DevModeData += struct.calcsize("H") * '\x00' + dwFields = (xres and DM_PELSWIDTH or 0) | (yres and DM_PELSHEIGHT or 0) | (BitsPerPixel and DM_BITSPERPEL or 0) + DevModeData += struct.pack("L", dwFields) + DevModeData += struct.calcsize("l9h32BHL") * '\x00' + DevModeData += struct.pack("LLL", BitsPerPixel or 0, xres or 0, yres or 0) + DevModeData += struct.calcsize("8L") * '\x00' + + return windll.user32.ChangeDisplaySettingsA(DevModeData, 0) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/testing/mozharness/external_tools/performance-artifact-schema.json b/testing/mozharness/external_tools/performance-artifact-schema.json new file mode 100644 index 000000000..f79a0419b --- /dev/null +++ b/testing/mozharness/external_tools/performance-artifact-schema.json @@ -0,0 +1,164 @@ +{ + "definitions": { + "framework_schema": { + "properties": { + "name": { + "title": "Framework name", + "type": "string" + } + }, + "type": "object" + }, + "subtest_schema": { + "properties": { + "name": { + "title": "Subtest name", + "type": "string" + }, + "value": { + "description": "Summary value for subtest", + "title": "Subtest value", + "type": "number", + "minimum": -1000000000000.0, + "maximum": 1000000000000.0 + }, + "lowerIsBetter": { + "description": "Whether lower values are better for subtest", + "title": "Lower is better", + "type": "boolean" + }, + "shouldAlert": { + "description": "Whether we should alert", + "title": "Should alert", + "type": "boolean" + }, + "alertThreshold": { + "description": "% change threshold before alerting", + "title": "Alert threshold", + "type": "number", + "minimum": 0.0, + "maximum": 1000.0 + }, + "minBackWindow": { + "description": "Minimum back window to use for alerting", + "title": "Minimum back window", + "type": "number", + "minimum": 1, + "maximum": 255 + }, + "maxBackWindow": { + "description": "Maximum back window to use for alerting", + "title": "Maximum back window", + "type": "number", + "minimum": 1, + "maximum": 255 + }, + "foreWindow": { + "description": "Fore window to use for alerting", + "title": "Fore window", + "type": "number", + "minimum": 1, + "maximum": 255 + } + }, + "required": [ + "name", + "value" + ], + "type": "object" + }, + "suite_schema": { + "properties": { + "name": { + "title": "Suite name", + "type": "string" + }, + "extraOptions": { + "type": "array", + "title": "Extra options used in running suite", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "subtests": { + "items": { + "$ref": "#/definitions/subtest_schema" + }, + "title": "Subtests", + "type": "array" + }, + "value": { + "title": "Suite value", + "type": "number", + "minimum": -1000000000000.0, + "maximum": 1000000000000.0 + }, + "lowerIsBetter": { + "description": "Whether lower values are better for suite", + "title": "Lower is better", + "type": "boolean" + }, + "shouldAlert": { + "description": "Whether we should alert on this suite (overrides default behaviour)", + "title": "Should alert", + "type": "boolean" + }, + "alertThreshold": { + "description": "% change threshold before alerting", + "title": "Alert threshold", + "type": "number", + "minimum": 0.0, + "maximum": 1000.0 + }, + "minBackWindow": { + "description": "Minimum back window to use for alerting", + "title": "Minimum back window", + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "maxBackWindow": { + "description": "Maximum back window to use for alerting", + "title": "Maximum back window", + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "foreWindow": { + "description": "Fore window to use for alerting", + "title": "Fore window", + "type": "integer", + "minimum": 1, + "maximum": 255 + } + }, + "required": [ + "name", + "subtests" + ], + "type": "object" + } + }, + "description": "Structure for submitting performance data as part of a job", + "id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#", + "properties": { + "framework": { + "$ref": "#/definitions/framework_schema" + }, + "suites": { + "description": "List of suite-level data submitted as part of this structure", + "items": { + "$ref": "#/definitions/suite_schema" + }, + "title": "Performance suites", + "type": "array" + } + }, + "required": [ + "framework", + "suites" + ], + "title": "Perfherder Schema", + "type": "object" +} diff --git a/testing/mozharness/external_tools/robustcheckout.py b/testing/mozharness/external_tools/robustcheckout.py new file mode 100644 index 000000000..e801724c1 --- /dev/null +++ b/testing/mozharness/external_tools/robustcheckout.py @@ -0,0 +1,451 @@ +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +"""Robustly perform a checkout. + +This extension provides the ``hg robustcheckout`` command for +ensuring a working directory is updated to the specified revision +from a source repo using best practices to ensure optimal clone +times and storage efficiency. +""" + +from __future__ import absolute_import + +import contextlib +import errno +import functools +import os +import random +import re +import socket +import ssl +import time +import urllib2 + +from mercurial.i18n import _ +from mercurial.node import hex +from mercurial import ( + commands, + error, + exchange, + extensions, + cmdutil, + hg, + registrar, + scmutil, + util, +) + +testedwith = '3.7 3.8 3.9 4.0 4.1 4.2 4.3' +minimumhgversion = '3.7' + +cmdtable = {} + +# Mercurial 4.3 introduced registrar.command as a replacement for +# cmdutil.command. +if util.safehasattr(registrar, 'command'): + command = registrar.command(cmdtable) +else: + command = cmdutil.command(cmdtable) + +# Mercurial 4.2 introduced the vfs module and deprecated the symbol in +# scmutil. +def getvfs(): + try: + from mercurial.vfs import vfs + return vfs + except ImportError: + return scmutil.vfs + + +if os.name == 'nt': + import ctypes + + # Get a reference to the DeleteFileW function + # DeleteFileW accepts filenames encoded as a null terminated sequence of + # wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode + # strings to null terminated UTF-16 strings. + # However, we receive (byte) strings from mercurial. When these are passed + # to DeleteFileW via the c_wchar_p type, they are implicitly decoded via + # the 'mbcs' encoding on windows. + kernel32 = ctypes.windll.kernel32 + DeleteFile = kernel32.DeleteFileW + DeleteFile.argtypes = [ctypes.c_wchar_p] + DeleteFile.restype = ctypes.c_bool + + def unlinklong(fn): + normalized_path = '\\\\?\\' + os.path.normpath(fn) + if not DeleteFile(normalized_path): + raise OSError(errno.EPERM, "couldn't remove long path", fn) + +# Not needed on other platforms, but is handy for testing +else: + def unlinklong(fn): + os.unlink(fn) + + +def unlinkwrapper(unlinkorig, fn, ui): + '''Calls unlink_long if original unlink function fails.''' + try: + ui.debug('calling unlink_orig %s\n' % fn) + return unlinkorig(fn) + except WindowsError as e: + # Windows error 3 corresponds to ERROR_PATH_NOT_FOUND + # only handle this case; re-raise the exception for other kinds of + # failures. + if e.winerror != 3: + raise + ui.debug('caught WindowsError ERROR_PATH_NOT_FOUND; ' + 'calling unlink_long %s\n' % fn) + return unlinklong(fn) + + +@contextlib.contextmanager +def wrapunlink(ui): + '''Context manager that temporarily monkeypatches unlink functions.''' + purgemod = extensions.find('purge') + to_wrap = [(purgemod.util, 'unlink')] + + # Pass along the ui object to the unlink_wrapper so we can get logging out + # of it. + wrapped = functools.partial(unlinkwrapper, ui=ui) + + # Wrap the original function(s) with our unlink wrapper. + originals = {} + for mod, func in to_wrap: + ui.debug('wrapping %s %s\n' % (mod, func)) + originals[mod, func] = extensions.wrapfunction(mod, func, wrapped) + + try: + yield + finally: + # Restore the originals. + for mod, func in to_wrap: + ui.debug('restoring %s %s\n' % (mod, func)) + setattr(mod, func, originals[mod, func]) + + +def purgewrapper(orig, ui, *args, **kwargs): + '''Runs original purge() command with unlink monkeypatched.''' + with wrapunlink(ui): + return orig(ui, *args, **kwargs) + + +@command('robustcheckout', [ + ('', 'upstream', '', 'URL of upstream repo to clone from'), + ('r', 'revision', '', 'Revision to check out'), + ('b', 'branch', '', 'Branch to check out'), + ('', 'purge', False, 'Whether to purge the working directory'), + ('', 'sharebase', '', 'Directory where shared repos should be placed'), + ('', 'networkattempts', 3, 'Maximum number of attempts for network ' + 'operations'), + ], + '[OPTION]... URL DEST', + norepo=True) +def robustcheckout(ui, url, dest, upstream=None, revision=None, branch=None, + purge=False, sharebase=None, networkattempts=None): + """Ensure a working copy has the specified revision checked out.""" + if not revision and not branch: + raise error.Abort('must specify one of --revision or --branch') + + if revision and branch: + raise error.Abort('cannot specify both --revision and --branch') + + # Require revision to look like a SHA-1. + if revision: + if len(revision) < 12 or len(revision) > 40 or not re.match('^[a-f0-9]+$', revision): + raise error.Abort('--revision must be a SHA-1 fragment 12-40 ' + 'characters long') + + sharebase = sharebase or ui.config('share', 'pool') + if not sharebase: + raise error.Abort('share base directory not defined; refusing to operate', + hint='define share.pool config option or pass --sharebase') + + # worker.backgroundclose only makes things faster if running anti-virus, + # which our automation doesn't. Disable it. + ui.setconfig('worker', 'backgroundclose', False) + + # By default the progress bar starts after 3s and updates every 0.1s. We + # change this so it shows and updates every 1.0s. + # We also tell progress to assume a TTY is present so updates are printed + # even if there is no known TTY. + # We make the config change here instead of in a config file because + # otherwise we're at the whim of whatever configs are used in automation. + ui.setconfig('progress', 'delay', 1.0) + ui.setconfig('progress', 'refresh', 1.0) + ui.setconfig('progress', 'assume-tty', True) + + sharebase = os.path.realpath(sharebase) + + return _docheckout(ui, url, dest, upstream, revision, branch, purge, + sharebase, networkattempts) + +def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase, + networkattemptlimit, networkattempts=None): + if not networkattempts: + networkattempts = [1] + + def callself(): + return _docheckout(ui, url, dest, upstream, revision, branch, purge, + sharebase, networkattemptlimit, networkattempts) + + ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch, + dest)) + + # We assume that we're the only process on the machine touching the + # repository paths that we were told to use. This means our recovery + # scenario when things aren't "right" is to just nuke things and start + # from scratch. This is easier to implement than verifying the state + # of the data and attempting recovery. And in some scenarios (such as + # potential repo corruption), it is probably faster, since verifying + # repos can take a while. + + destvfs = getvfs()(dest, audit=False, realpath=True) + + def deletesharedstore(path=None): + storepath = path or destvfs.read('.hg/sharedpath').strip() + if storepath.endswith('.hg'): + storepath = os.path.dirname(storepath) + + storevfs = getvfs()(storepath, audit=False) + storevfs.rmtree(forcibly=True) + + if destvfs.exists() and not destvfs.exists('.hg'): + raise error.Abort('destination exists but no .hg directory') + + # Require checkouts to be tied to shared storage because efficiency. + if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'): + ui.warn('(destination is not shared; deleting)\n') + destvfs.rmtree(forcibly=True) + + # Verify the shared path exists and is using modern pooled storage. + if destvfs.exists('.hg/sharedpath'): + storepath = destvfs.read('.hg/sharedpath').strip() + + ui.write('(existing repository shared store: %s)\n' % storepath) + + if not os.path.exists(storepath): + ui.warn('(shared store does not exist; deleting destination)\n') + destvfs.rmtree(forcibly=True) + elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')): + ui.warn('(shared store does not belong to pooled storage; ' + 'deleting destination to improve efficiency)\n') + destvfs.rmtree(forcibly=True) + + storevfs = getvfs()(storepath, audit=False) + if storevfs.isfileorlink('store/lock'): + ui.warn('(shared store has an active lock; assuming it is left ' + 'over from a previous process and that the store is ' + 'corrupt; deleting store and destination just to be ' + 'sure)\n') + destvfs.rmtree(forcibly=True) + deletesharedstore(storepath) + + # FUTURE when we require generaldelta, this is where we can check + # for that. + + if destvfs.isfileorlink('.hg/wlock'): + ui.warn('(dest has an active working directory lock; assuming it is ' + 'left over from a previous process and that the destination ' + 'is corrupt; deleting it just to be sure)\n') + destvfs.rmtree(forcibly=True) + + def handlerepoerror(e): + if e.message == _('abandoned transaction found'): + ui.warn('(abandoned transaction found; trying to recover)\n') + repo = hg.repository(ui, dest) + if not repo.recover(): + ui.warn('(could not recover repo state; ' + 'deleting shared store)\n') + deletesharedstore() + + ui.warn('(attempting checkout from beginning)\n') + return callself() + + raise + + # At this point we either have an existing working directory using + # shared, pooled storage or we have nothing. + + def handlenetworkfailure(): + if networkattempts[0] >= networkattemptlimit: + raise error.Abort('reached maximum number of network attempts; ' + 'giving up\n') + + ui.warn('(retrying after network failure on attempt %d of %d)\n' % + (networkattempts[0], networkattemptlimit)) + + # Do a backoff on retries to mitigate the thundering herd + # problem. This is an exponential backoff with a multipler + # plus random jitter thrown in for good measure. + # With the default settings, backoffs will be: + # 1) 2.5 - 6.5 + # 2) 5.5 - 9.5 + # 3) 11.5 - 15.5 + backoff = (2 ** networkattempts[0] - 1) * 1.5 + jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000) + jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000) + backoff += float(random.randint(jittermin, jittermax)) / 1000.0 + ui.warn('(waiting %.2fs before retry)\n' % backoff) + time.sleep(backoff) + + networkattempts[0] += 1 + + def handlepullerror(e): + """Handle an exception raised during a pull. + + Returns True if caller should call ``callself()`` to retry. + """ + if isinstance(e, error.Abort): + if e.args[0] == _('repository is unrelated'): + ui.warn('(repository is unrelated; deleting)\n') + destvfs.rmtree(forcibly=True) + return True + elif e.args[0].startswith(_('stream ended unexpectedly')): + ui.warn('%s\n' % e.args[0]) + # Will raise if failure limit reached. + handlenetworkfailure() + return True + elif isinstance(e, ssl.SSLError): + # Assume all SSL errors are due to the network, as Mercurial + # should convert non-transport errors like cert validation failures + # to error.Abort. + ui.warn('ssl error: %s\n' % e) + handlenetworkfailure() + return True + elif isinstance(e, urllib2.URLError): + if isinstance(e.reason, socket.error): + ui.warn('socket error: %s\n' % e.reason) + handlenetworkfailure() + return True + + return False + + created = False + + if not destvfs.exists(): + # Ensure parent directories of destination exist. + # Mercurial 3.8 removed ensuredirs and made makedirs race safe. + if util.safehasattr(util, 'ensuredirs'): + makedirs = util.ensuredirs + else: + makedirs = util.makedirs + + makedirs(os.path.dirname(destvfs.base), notindexed=True) + makedirs(sharebase, notindexed=True) + + if upstream: + ui.write('(cloning from upstream repo %s)\n' % upstream) + cloneurl = upstream or url + + try: + res = hg.clone(ui, {}, cloneurl, dest=dest, update=False, + shareopts={'pool': sharebase, 'mode': 'identity'}) + except (error.Abort, ssl.SSLError, urllib2.URLError) as e: + if handlepullerror(e): + return callself() + raise + except error.RepoError as e: + return handlerepoerror(e) + except error.RevlogError as e: + ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message) + deletesharedstore() + return callself() + + # TODO retry here. + if res is None: + raise error.Abort('clone failed') + + # Verify it is using shared pool storage. + if not destvfs.exists('.hg/sharedpath'): + raise error.Abort('clone did not create a shared repo') + + created = True + + # The destination .hg directory should exist. Now make sure we have the + # wanted revision. + + repo = hg.repository(ui, dest) + + # We only pull if we are using symbolic names or the requested revision + # doesn't exist. + havewantedrev = False + if revision and revision in repo: + ctx = repo[revision] + + if not ctx.hex().startswith(revision): + raise error.Abort('--revision argument is ambiguous', + hint='must be the first 12+ characters of a ' + 'SHA-1 fragment') + + checkoutrevision = ctx.hex() + havewantedrev = True + + if not havewantedrev: + ui.write('(pulling to obtain %s)\n' % (revision or branch,)) + + remote = None + try: + remote = hg.peer(repo, {}, url) + pullrevs = [remote.lookup(revision or branch)] + checkoutrevision = hex(pullrevs[0]) + if branch: + ui.warn('(remote resolved %s to %s; ' + 'result is not deterministic)\n' % + (branch, checkoutrevision)) + + if checkoutrevision in repo: + ui.warn('(revision already present locally; not pulling)\n') + else: + pullop = exchange.pull(repo, remote, heads=pullrevs) + if not pullop.rheads: + raise error.Abort('unable to pull requested revision') + except (error.Abort, ssl.SSLError, urllib2.URLError) as e: + if handlepullerror(e): + return callself() + raise + except error.RepoError as e: + return handlerepoerror(e) + except error.RevlogError as e: + ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message) + deletesharedstore() + return callself() + finally: + if remote: + remote.close() + + # Now we should have the wanted revision in the store. Perform + # working directory manipulation. + + # Purge if requested. We purge before update because this way we're + # guaranteed to not have conflicts on `hg update`. + if purge and not created: + ui.write('(purging working directory)\n') + purgeext = extensions.find('purge') + + if purgeext.purge(ui, repo, all=True, abort_on_err=True, + # The function expects all arguments to be + # defined. + **{'print': None, 'print0': None, 'dirs': None, + 'files': None}): + raise error.Abort('error purging') + + # Update the working directory. + if commands.update(ui, repo, rev=checkoutrevision, clean=True): + raise error.Abort('error updating') + + ui.write('updated to %s\n' % checkoutrevision) + return None + + +def extsetup(ui): + # Ensure required extensions are loaded. + for ext in ('purge', 'share'): + try: + extensions.find(ext) + except KeyError: + extensions.load(ui, ext, None) + + purgemod = extensions.find('purge') + extensions.wrapcommand(purgemod.cmdtable, 'purge', purgewrapper) diff --git a/testing/mozharness/external_tools/virtualenv/AUTHORS.txt b/testing/mozharness/external_tools/virtualenv/AUTHORS.txt new file mode 100644 index 000000000..272494163 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/AUTHORS.txt @@ -0,0 +1,91 @@ +Author +------ + +Ian Bicking + +Maintainers +----------- + +Brian Rosner +Carl Meyer +Jannis Leidel +Paul Moore +Paul Nasrat +Marcus Smith + +Contributors +------------ + +Alex Grönholm +Anatoly Techtonik +Antonio Cuni +Antonio Valentino +Armin Ronacher +Barry Warsaw +Benjamin Root +Bradley Ayers +Branden Rolston +Brandon Carl +Brian Kearns +Cap Petschulat +CBWhiz +Chris Adams +Chris McDonough +Christos Kontas +Christian Hudon +Christian Stefanescu +Christopher Nilsson +Cliff Xuan +Curt Micol +Damien Nozay +Dan Sully +Daniel Hahler +Daniel Holth +David Schoonover +Denis Costa +Doug Hellmann +Doug Napoleone +Douglas Creager +Eduard-Cristian Stefan +Erik M. Bray +Ethan Jucovy +Gabriel de Perthuis +Gunnlaugur Thor Briem +Graham Dennis +Greg Haskins +Jason Penney +Jason R. Coombs +Jeff Hammel +Jeremy Orem +Jason Penney +Jason R. Coombs +John Kleint +Jonathan Griffin +Jonathan Hitchcock +Jorge Vargas +Josh Bronson +Kamil Kisiel +Kyle Gibson +Konstantin Zemlyak +Kumar McMillan +Lars Francke +Marc Abramowitz +Mika Laitio +Mike Hommey +Miki Tebeka +Philip Jenvey +Philippe Ombredanne +Piotr Dobrogost +Preston Holmes +Ralf Schmitt +Raul Leal +Ronny Pfannschmidt +Satrajit Ghosh +Sergio de Carvalho +Stefano Rivera +Tarek Ziadé +Thomas Aglassinger +Vinay Sajip +Vitaly Babiy +Vladimir Rutsky +Wang Xuerui
\ No newline at end of file diff --git a/testing/mozharness/external_tools/virtualenv/LICENSE.txt b/testing/mozharness/external_tools/virtualenv/LICENSE.txt new file mode 100644 index 000000000..ab145001f --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2007 Ian Bicking and Contributors +Copyright (c) 2009 Ian Bicking, The Open Planning Project +Copyright (c) 2011-2016 The virtualenv developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/testing/mozharness/external_tools/virtualenv/MANIFEST.in b/testing/mozharness/external_tools/virtualenv/MANIFEST.in new file mode 100644 index 000000000..49037ada6 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/MANIFEST.in @@ -0,0 +1,12 @@ +recursive-include docs * +recursive-include tests *.py *.sh *.expected +recursive-include virtualenv_support *.whl +recursive-include virtualenv_embedded * +recursive-exclude docs/_templates * +recursive-exclude docs/_build * +include virtualenv_support/__init__.py +include bin/* +include scripts/* +include *.py +include AUTHORS.txt +include LICENSE.txt diff --git a/testing/mozharness/external_tools/virtualenv/PKG-INFO b/testing/mozharness/external_tools/virtualenv/PKG-INFO new file mode 100644 index 000000000..dbfda645d --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/PKG-INFO @@ -0,0 +1,87 @@ +Metadata-Version: 1.1 +Name: virtualenv +Version: 15.0.1 +Summary: Virtual Python Environment builder +Home-page: https://virtualenv.pypa.io/ +Author: Jannis Leidel, Carl Meyer and Brian Rosner +Author-email: python-virtualenv@groups.google.com +License: MIT +Description: Virtualenv + ========== + + `Mailing list <http://groups.google.com/group/python-virtualenv>`_ | + `Issues <https://github.com/pypa/virtualenv/issues>`_ | + `Github <https://github.com/pypa/virtualenv>`_ | + `PyPI <https://pypi.python.org/pypi/virtualenv/>`_ | + User IRC: #pypa + Dev IRC: #pypa-dev + + Introduction + ------------ + + ``virtualenv`` is a tool to create isolated Python environments. + + The basic problem being addressed is one of dependencies and versions, + and indirectly permissions. Imagine you have an application that + needs version 1 of LibFoo, but another application requires version + 2. How can you use both these applications? If you install + everything into ``/usr/lib/python2.7/site-packages`` (or whatever your + platform's standard location is), it's easy to end up in a situation + where you unintentionally upgrade an application that shouldn't be + upgraded. + + Or more generally, what if you want to install an application *and + leave it be*? If an application works, any change in its libraries or + the versions of those libraries can break the application. + + Also, what if you can't install packages into the global + ``site-packages`` directory? For instance, on a shared host. + + In all these cases, ``virtualenv`` can help you. It creates an + environment that has its own installation directories, that doesn't + share libraries with other virtualenv environments (and optionally + doesn't access the globally installed libraries either). + + .. comment: + + Release History + =============== + + 15.0.1 (2016-03-17) + ------------------- + + * Print error message when DEST_DIR exists and is a file + + * Upgrade setuptools to 20.3 + + * Upgrade pip to 8.1.1. + + + 15.0.0 (2016-03-05) + ------------------- + + * Remove the `virtualenv-N.N` script from the package; this can no longer be + correctly created from a wheel installation. + Resolves #851, #692 + + * Remove accidental runtime dependency on pip by extracting certificate in the + subprocess. + + * Upgrade setuptools 20.2.2. + + * Upgrade pip to 8.1.0. + + + `Full Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_. +Keywords: setuptools deployment installation distutils +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 diff --git a/testing/mozharness/external_tools/virtualenv/README.rst b/testing/mozharness/external_tools/virtualenv/README.rst new file mode 100644 index 000000000..0d5984dce --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/README.rst @@ -0,0 +1,31 @@ +virtualenv +========== + +A tool for creating isolated 'virtual' python environments. + +.. image:: https://img.shields.io/pypi/v/virtualenv.svg + :target: https://pypi.python.org/pypi/virtualenv + +.. image:: https://img.shields.io/travis/pypa/virtualenv/develop.svg + :target: http://travis-ci.org/pypa/virtualenv + +* `Installation <https://virtualenv.pypa.io/en/latest/installation.html>`_ +* `Documentation <https://virtualenv.pypa.io/>`_ +* `Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_ +* `Issues <https://github.com/pypa/virtualenv/issues>`_ +* `PyPI <https://pypi.python.org/pypi/virtualenv/>`_ +* `Github <https://github.com/pypa/virtualenv>`_ +* `User mailing list <http://groups.google.com/group/python-virtualenv>`_ +* `Dev mailing list <http://groups.google.com/group/pypa-dev>`_ +* User IRC: #pypa on Freenode. +* Dev IRC: #pypa-dev on Freenode. + + +Code of Conduct +--------------- + +Everyone interacting in the virtualenv project's codebases, issue trackers, +chat rooms, and mailing lists is expected to follow the +`PyPA Code of Conduct`_. + +.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ diff --git a/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py b/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py new file mode 100755 index 000000000..a816af3eb --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +""" +Helper script to rebuild virtualenv.py from virtualenv_support +""" +from __future__ import print_function + +import os +import re +import codecs +from zlib import crc32 + +here = os.path.dirname(__file__) +script = os.path.join(here, '..', 'virtualenv.py') + +gzip = codecs.lookup('zlib') +b64 = codecs.lookup('base64') + +file_regex = re.compile( + br'##file (.*?)\n([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*convert\("""\n(.*?)"""\)', + re.S) +file_template = b'##file %(filename)s\n%(varname)s = convert("""\n%(data)s""")' + +def rebuild(script_path): + with open(script_path, 'rb') as f: + script_content = f.read() + parts = [] + last_pos = 0 + match = None + for match in file_regex.finditer(script_content): + parts += [script_content[last_pos:match.start()]] + last_pos = match.end() + filename, fn_decoded = match.group(1), match.group(1).decode() + varname = match.group(2) + data = match.group(3) + + print('Found file %s' % fn_decoded) + pathname = os.path.join(here, '..', 'virtualenv_embedded', fn_decoded) + + with open(pathname, 'rb') as f: + embedded = f.read() + new_crc = crc32(embedded) + new_data = b64.encode(gzip.encode(embedded)[0])[0] + + if new_data == data: + print(' File up to date (crc: %s)' % new_crc) + parts += [match.group(0)] + continue + # Else: content has changed + crc = crc32(gzip.decode(b64.decode(data)[0])[0]) + print(' Content changed (crc: %s -> %s)' % + (crc, new_crc)) + new_match = file_template % { + b'filename': filename, + b'varname': varname, + b'data': new_data + } + parts += [new_match] + + parts += [script_content[last_pos:]] + new_content = b''.join(parts) + + if new_content != script_content: + print('Content updated; overwriting... ', end='') + with open(script_path, 'wb') as f: + f.write(new_content) + print('done.') + else: + print('No changes in content') + if match is None: + print('No variables were matched/found') + +if __name__ == '__main__': + rebuild(script) diff --git a/testing/mozharness/external_tools/virtualenv/docs/Makefile b/testing/mozharness/external_tools/virtualenv/docs/Makefile new file mode 100644 index 000000000..e4de9f847 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/Makefile @@ -0,0 +1,130 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-compressor.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-compressor.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/django-compressor" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-compressor" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + make -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/testing/mozharness/external_tools/virtualenv/docs/changes.rst b/testing/mozharness/external_tools/virtualenv/docs/changes.rst new file mode 100644 index 000000000..2df19f666 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/changes.rst @@ -0,0 +1,985 @@ +Release History +=============== + +15.0.1 (2016-03-17) +------------------- + +* Print error message when DEST_DIR exists and is a file + +* Upgrade setuptools to 20.3 + +* Upgrade pip to 8.1.1. + + +15.0.0 (2016-03-05) +------------------- + +* Remove the `virtualenv-N.N` script from the package; this can no longer be + correctly created from a wheel installation. + Resolves :issue:`851`, :issue:`692` + +* Remove accidental runtime dependency on pip by extracting certificate in the + subprocess. + +* Upgrade setuptools 20.2.2. + +* Upgrade pip to 8.1.0. + + +14.0.6 (2016-02-07) +------------------- + +* Upgrade setuptools to 20.0 + +* Upgrade wheel to 0.29.0 + +* Fix an error where virtualenv didn't pass in a working ssl certificate for + pip, causing "weird" errors related to ssl. + + +14.0.5 (2016-02-01) +------------------- + +* Homogenize drive letter casing for both prefixes and filenames. :issue:`858` + + +14.0.4 (2016-01-31) +------------------- + +* Upgrade setuptools to 19.6.2 + +* Revert ac4ea65; only correct drive letter case. + Fixes :issue:`856`, :issue:`815` + + +14.0.3 (2016-01-28) +------------------- + +* Upgrade setuptools to 19.6.1 + + +14.0.2 (2016-01-28) +------------------- + +* Upgrade setuptools to 19.6 + +* Supress any errors from `unset` on different shells (:pull:`843`) + +* Normalize letter case for prefix path checking. Fixes :issue:`837` + + +14.0.1 (2016-01-21) +------------------- + +* Upgrade from pip 8.0.0 to 8.0.2. + +* Fix the default of ``--(no-)download`` to default to downloading. + + +14.0.0 (2016-01-19) +------------------- + +* **BACKWARDS INCOMPATIBLE** Drop support for Python 3.2. + +* Upgrade setuptools to 19.4 + +* Upgrade wheel to 0.26.0 + +* Upgrade pip to 8.0.0 + +* Upgrade argparse to 1.4.0 + +* Added support for ``python-config`` script (:pull:`798`) + +* Updated activate.fish (:pull:`589`) (:pull:`799`) + +* Account for a ``site.pyo`` correctly in some python implementations (:pull:`759`) + +* Properly restore an empty PS1 (:issue:`407`) + +* Properly remove ``pydoc`` when deactivating + +* Remove workaround for very old Mageia / Mandriva linuxes (:pull:`472`) + +* Added a space after virtualenv name in the prompt: ``(env) $PS1`` + +* Make sure not to run a --user install when creating the virtualenv (:pull:`803`) + +* Remove virtualenv.py's path from sys.path when executing with a new + python. Fixes issue :issue:`779`, :issue:`763` (:pull:`805`) + +* Remove use of () in .bat files so ``Program Files (x86)`` works :issue:`35` + +* Download new releases of the preinstalled software from PyPI when there are + new releases available. This behavior can be disabled using + ``--no-download``. + +* Make ``--no-setuptools``, ``--no-pip``, and ``--no-wheel`` independent of + each other. + + +13.1.2 (2015-08-23) +------------------- + +* Upgrade pip to 7.1.2. + + +13.1.1 (2015-08-20) +------------------- + +* Upgrade pip to 7.1.1. + +* Upgrade setuptools to 18.2. + +* Make the activate script safe to use when bash is running with ``-u``. + + +13.1.0 (2015-06-30) +------------------- + +* Upgrade pip to 7.1.0 + +* Upgrade setuptools to 18.0.1 + + +13.0.3 (2015-06-01) +------------------- + +* Upgrade pip to 7.0.3 + + +13.0.2 (2015-06-01) +------------------- + +* Upgrade pip to 7.0.2 + +* Upgrade setuptools to 17.0 + + +13.0.1 (2015-05-22) +------------------- + +* Upgrade pip to 7.0.1 + + +13.0.0 (2015-05-21) +------------------- + +* Automatically install wheel when creating a new virutalenv. This can be + disabled by using the ``--no-wheel`` option. + +* Don't trust the current directory as a location to discover files to install + packages from. + +* Upgrade setuptools to 16.0. + +* Upgrade pip to 7.0.0. + + +12.1.1 (2015-04-07) +------------------- + +* Upgrade pip to 6.1.1 + + +12.1.0 (2015-04-07) +------------------- + +* Upgrade setuptools to 15.0 + +* Upgrade pip to 6.1.0 + + +12.0.7 (2015-02-04) +------------------- + +* Upgrade pip to 6.0.8 + + +12.0.6 (2015-01-28) +------------------- + +* Upgrade pip to 6.0.7 + +* Upgrade setuptools to 12.0.5 + + +12.0.5 (2015-01-03) +------------------- + +* Upgrade pip to 6.0.6 + +* Upgrade setuptools to 11.0 + + +12.0.4 (2014-12-23) +------------------- + +* Revert the fix to ``-p`` on Debian based pythons as it was broken in other + situations. + +* Revert several sys.path changes new in 12.0 which were breaking virtualenv. + +12.0.3 (2014-12-23) +------------------- + +* Fix an issue where Debian based Pythons would fail when using -p with the + host Python. + +* Upgrade pip to 6.0.3 + +12.0.2 (2014-12-23) +------------------- + +* Upgraded pip to 6.0.2 + +12.0.1 (2014-12-22) +------------------- + +* Upgraded pip to 6.0.1 + + +12.0 (2014-12-22) +----------------- + +* **PROCESS** Version numbers are now simply ``X.Y`` where the leading ``1`` + has been dropped. +* Split up documentation into structured pages +* Now using pytest framework +* Correct sys.path ordering for debian, issue #461 +* Correctly throws error on older Pythons, issue #619 +* Allow for empty $PATH, pull #601 +* Don't set prompt if $env:VIRTUAL_ENV_DISABLE_PROMPT is set for Powershell +* Updated setuptools to 7.0 + +1.11.6 (2014-05-16) +------------------- + +* Updated setuptools to 3.6 +* Updated pip to 1.5.6 + +1.11.5 (2014-05-03) +------------------- + +* Updated setuptools to 3.4.4 +* Updated documentation to use https://virtualenv.pypa.io/ +* Updated pip to 1.5.5 + +1.11.4 (2014-02-21) +------------------- + +* Updated pip to 1.5.4 + + +1.11.3 (2014-02-20) +------------------- + +* Updated setuptools to 2.2 +* Updated pip to 1.5.3 + + +1.11.2 (2014-01-26) +------------------- + +* Fixed easy_install installed virtualenvs by updated pip to 1.5.2 + +1.11.1 (2014-01-20) +------------------- + +* Fixed an issue where pip and setuptools were not getting installed when using + the ``--system-site-packages`` flag. +* Updated setuptools to fix an issue when installed with easy_install +* Fixed an issue with Python 3.4 and sys.stdout encoding being set to ascii +* Upgraded pip to v1.5.1 +* Upgraded setuptools to v2.1 + +1.11 (2014-01-02) +----------------- + +* **BACKWARDS INCOMPATIBLE** Switched to using wheels for the bundled copies of + setuptools and pip. Using sdists is no longer supported - users supplying + their own versions of pip/setuptools will need to provide wheels. +* **BACKWARDS INCOMPATIBLE** Modified the handling of ``--extra-search-dirs``. + This option now works like pip's ``--find-links`` option, in that it adds + extra directories to search for compatible wheels for pip and setuptools. + The actual wheel selected is chosen based on version and compatibility, using + the same algorithm as ``pip install setuptools``. +* Fixed #495, --always-copy was failing (#PR 511) +* Upgraded pip to v1.5 +* Upgraded setuptools to v1.4 + +1.10.1 (2013-08-07) +------------------- + +* **New Signing Key** Release 1.10.1 is using a different key than normal with + fingerprint: 7C6B 7C5D 5E2B 6356 A926 F04F 6E3C BCE9 3372 DCFA +* Upgraded pip to v1.4.1 +* Upgraded setuptools to v0.9.8 + + +1.10 (2013-07-23) +----------------- + +* **BACKWARDS INCOMPATIBLE** Dropped support for Python 2.5. The minimum + supported Python version is now Python 2.6. + +* **BACKWARDS INCOMPATIBLE** Using ``virtualenv.py`` as an isolated script + (i.e. without an associated ``virtualenv_support`` directory) is no longer + supported for security reasons and will fail with an error. + + Along with this, ``--never-download`` is now always pinned to ``True``, and + is only being maintained in the short term for backward compatibility + (Pull #412). + +* **IMPORTANT** Switched to the new setuptools (v0.9.7) which has been merged + with Distribute_ again and works for Python 2 and 3 with one codebase. + The ``--distribute`` and ``--setuptools`` options are now no-op. + +* Updated to pip 1.4. + +* Added support for PyPy3k + +* Added the option to use a version number with the ``-p`` option to get the + system copy of that Python version (Windows only) + +* Removed embedded ``ez_setup.py``, ``distribute_setup.py`` and + ``distribute_from_egg.py`` files as part of switching to merged setuptools. + +* Fixed ``--relocatable`` to work better on Windows. + +* Fixed issue with readline on Windows. + +.. _Distribute: https://pypi.python.org/pypi/distribute + +1.9.1 (2013-03-08) +------------------ + +* Updated to pip 1.3.1 that fixed a major backward incompatible change of + parsing URLs to externally hosted packages that got accidentily included + in pip 1.3. + +1.9 (2013-03-07) +---------------- + +* Unset VIRTUAL_ENV environment variable in deactivate.bat (Pull #364) +* Upgraded distribute to 0.6.34. +* Added ``--no-setuptools`` and ``--no-pip`` options (Pull #336). +* Fixed Issue #373. virtualenv-1.8.4 was failing in cygwin (Pull #382). +* Fixed Issue #378. virtualenv is now "multiarch" aware on debian/ubuntu (Pull #379). +* Fixed issue with readline module path on pypy and OSX (Pull #374). +* Made 64bit detection compatible with Python 2.5 (Pull #393). + + +1.8.4 (2012-11-25) +------------------ + +* Updated distribute to 0.6.31. This fixes #359 (numpy install regression) on + UTF-8 platforms, and provides a workaround on other platforms: + ``PYTHONIOENCODING=utf8 pip install numpy``. + +* When installing virtualenv via curl, don't forget to filter out arguments + the distribute setup script won't understand. Fixes #358. + +* Added some more integration tests. + +* Removed the unsupported embedded setuptools egg for Python 2.4 to reduce + file size. + +1.8.3 (2012-11-21) +------------------ + +* Fixed readline on OS X. Thanks minrk + +* Updated distribute to 0.6.30 (improves our error reporting, plus new + distribute features and fixes). Thanks Gabriel (g2p) + +* Added compatibility with multiarch Python (Python 3.3 for example). Added an + integration test. Thanks Gabriel (g2p) + +* Added ability to install distribute from a user-provided egg, rather than the + bundled sdist, for better speed. Thanks Paul Moore. + +* Make the creation of lib64 symlink smarter about already-existing symlink, + and more explicit about full paths. Fixes #334 and #330. Thanks Jeremy Orem. + +* Give lib64 site-dir preference over lib on 64-bit systems, to avoid wrong + 32-bit compiles in the venv. Fixes #328. Thanks Damien Nozay. + +* Fix a bug with prompt-handling in ``activate.csh`` in non-interactive csh + shells. Fixes #332. Thanks Benjamin Root for report and patch. + +* Make it possible to create a virtualenv from within a Python + 3.3. pyvenv. Thanks Chris McDonough for the report. + +* Add optional --setuptools option to be able to switch to it in case + distribute is the default (like in Debian). + +1.8.2 (2012-09-06) +------------------ + +* Updated the included pip version to 1.2.1 to fix regressions introduced + there in 1.2. + + +1.8.1 (2012-09-03) +------------------ + +* Fixed distribute version used with `--never-download`. Thanks michr for + report and patch. + +* Fix creating Python 3.3 based virtualenvs by unsetting the + ``__PYVENV_LAUNCHER__`` environment variable in subprocesses. + + +1.8 (2012-09-01) +---------------- + +* **Dropped support for Python 2.4** The minimum supported Python version is + now Python 2.5. + +* Fix `--relocatable` on systems that use lib64. Fixes #78. Thanks Branden + Rolston. + +* Symlink some additional modules under Python 3. Fixes #194. Thanks Vinay + Sajip, Ian Clelland, and Stefan Holek for the report. + +* Fix ``--relocatable`` when a script uses ``__future__`` imports. Thanks + Branden Rolston. + +* Fix a bug in the config option parser that prevented setting negative + options with environment variables. Thanks Ralf Schmitt. + +* Allow setting ``--no-site-packages`` from the config file. + +* Use ``/usr/bin/multiarch-platform`` if available to figure out the include + directory. Thanks for the patch, Mika Laitio. + +* Fix ``install_name_tool`` replacement to work on Python 3.X. + +* Handle paths of users' site-packages on Mac OS X correctly when changing + the prefix. + +* Updated the embedded version of distribute to 0.6.28 and pip to 1.2. + + +1.7.2 (2012-06-22) +------------------ + +* Updated to distribute 0.6.27. + +* Fix activate.fish on OS X. Fixes #8. Thanks David Schoonover. + +* Create a virtualenv-x.x script with the Python version when installing, so + virtualenv for multiple Python versions can be installed to the same + script location. Thanks Miki Tebeka. + +* Restored ability to create a virtualenv with a path longer than 78 + characters, without breaking creation of virtualenvs with non-ASCII paths. + Thanks, Bradley Ayers. + +* Added ability to create virtualenvs without having installed Apple's + developers tools (using an own implementation of ``install_name_tool``). + Thanks Mike Hommey. + +* Fixed PyPy and Jython support on Windows. Thanks Konstantin Zemlyak. + +* Added pydoc script to ease use. Thanks Marc Abramowitz. Fixes #149. + +* Fixed creating a bootstrap script on Python 3. Thanks Raul Leal. Fixes #280. + +* Fixed inconsistency when having set the ``PYTHONDONTWRITEBYTECODE`` env var + with the --distribute option or the ``VIRTUALENV_USE_DISTRIBUTE`` env var. + ``VIRTUALENV_USE_DISTRIBUTE`` is now considered again as a legacy alias. + + +1.7.1.2 (2012-02-17) +-------------------- + +* Fixed minor issue in `--relocatable`. Thanks, Cap Petschulat. + + +1.7.1.1 (2012-02-16) +-------------------- + +* Bumped the version string in ``virtualenv.py`` up, too. + +* Fixed rST rendering bug of long description. + + +1.7.1 (2012-02-16) +------------------ + +* Update embedded pip to version 1.1. + +* Fix `--relocatable` under Python 3. Thanks Doug Hellmann. + +* Added environ PATH modification to activate_this.py. Thanks Doug + Napoleone. Fixes #14. + +* Support creating virtualenvs directly from a Python build directory on + Windows. Thanks CBWhiz. Fixes #139. + +* Use non-recursive symlinks to fix things up for posix_local install + scheme. Thanks michr. + +* Made activate script available for use with msys and cygwin on Windows. + Thanks Greg Haskins, Cliff Xuan, Jonathan Griffin and Doug Napoleone. + Fixes #176. + +* Fixed creation of virtualenvs on Windows when Python is not installed for + all users. Thanks Anatoly Techtonik for report and patch and Doug + Napoleone for testing and confirmation. Fixes #87. + +* Fixed creation of virtualenvs using -p in installs where some modules + that ought to be in the standard library (e.g. `readline`) are actually + installed in `site-packages` next to `virtualenv.py`. Thanks Greg Haskins + for report and fix. Fixes #167. + +* Added activation script for Powershell (signed by Jannis Leidel). Many + thanks to Jason R. Coombs. + + +1.7 (2011-11-30) +---------------- + +* Gave user-provided ``--extra-search-dir`` priority over default dirs for + finding setuptools/distribute (it already had priority for finding pip). + Thanks Ethan Jucovy. + +* Updated embedded Distribute release to 0.6.24. Thanks Alex Gronholm. + +* Made ``--no-site-packages`` behavior the default behavior. The + ``--no-site-packages`` flag is still permitted, but displays a warning when + used. Thanks Chris McDonough. + +* New flag: ``--system-site-packages``; this flag should be passed to get the + previous default global-site-package-including behavior back. + +* Added ability to set command options as environment variables and options + in a ``virtualenv.ini`` file. + +* Fixed various encoding related issues with paths. Thanks Gunnlaugur Thor Briem. + +* Made ``virtualenv.py`` script executable. + + +1.6.4 (2011-07-21) +------------------ + +* Restored ability to run on Python 2.4, too. + + +1.6.3 (2011-07-16) +------------------ + +* Restored ability to run on Python < 2.7. + + +1.6.2 (2011-07-16) +------------------ + +* Updated embedded distribute release to 0.6.19. + +* Updated embedded pip release to 1.0.2. + +* Fixed #141 - Be smarter about finding pkg_resources when using the + non-default Python interpreter (by using the ``-p`` option). + +* Fixed #112 - Fixed path in docs. + +* Fixed #109 - Corrected doctests of a Logger method. + +* Fixed #118 - Fixed creating virtualenvs on platforms that use the + "posix_local" install scheme, such as Ubuntu with Python 2.7. + +* Add missing library to Python 3 virtualenvs (``_dummy_thread``). + + +1.6.1 (2011-04-30) +------------------ + +* Start to use git-flow. + +* Added support for PyPy 1.5 + +* Fixed #121 -- added sanity-checking of the -p argument. Thanks Paul Nasrat. + +* Added progress meter for pip installation as well as setuptools. Thanks Ethan + Jucovy. + +* Added --never-download and --search-dir options. Thanks Ethan Jucovy. + + +1.6 +--- + +* Added Python 3 support! Huge thanks to Vinay Sajip and Vitaly Babiy. + +* Fixed creation of virtualenvs on Mac OS X when standard library modules + (readline) are installed outside the standard library. + +* Updated bundled pip to 1.0. + + +1.5.2 +----- + +* Moved main repository to Github: https://github.com/pypa/virtualenv + +* Transferred primary maintenance from Ian to Jannis Leidel, Carl Meyer and Brian Rosner + +* Fixed a few more pypy related bugs. + +* Updated bundled pip to 0.8.2. + +* Handed project over to new team of maintainers. + +* Moved virtualenv to Github at https://github.com/pypa/virtualenv + + +1.5.1 +----- + +* Added ``_weakrefset`` requirement for Python 2.7.1. + +* Fixed Windows regression in 1.5 + + +1.5 +--- + +* Include pip 0.8.1. + +* Add support for PyPy. + +* Uses a proper temporary dir when installing environment requirements. + +* Add ``--prompt`` option to be able to override the default prompt prefix. + +* Fix an issue with ``--relocatable`` on Windows. + +* Fix issue with installing the wrong version of distribute. + +* Add fish and csh activate scripts. + + +1.4.9 +----- + +* Include pip 0.7.2 + + +1.4.8 +----- + +* Fix for Mac OS X Framework builds that use + ``--universal-archs=intel`` + +* Fix ``activate_this.py`` on Windows. + +* Allow ``$PYTHONHOME`` to be set, so long as you use ``source + bin/activate`` it will get unset; if you leave it set and do not + activate the environment it will still break the environment. + +* Include pip 0.7.1 + + +1.4.7 +----- + +* Include pip 0.7 + + +1.4.6 +----- + +* Allow ``activate.sh`` to skip updating the prompt (by setting + ``$VIRTUAL_ENV_DISABLE_PROMPT``). + + +1.4.5 +----- + +* Include pip 0.6.3 + +* Fix ``activate.bat`` and ``deactivate.bat`` under Windows when + ``PATH`` contained a parenthesis + + +1.4.4 +----- + +* Include pip 0.6.2 and Distribute 0.6.10 + +* Create the ``virtualenv`` script even when Setuptools isn't + installed + +* Fix problem with ``virtualenv --relocate`` when ``bin/`` has + subdirectories (e.g., ``bin/.svn/``); from Alan Franzoni. + +* If you set ``$VIRTUALENV_DISTRIBUTE`` then virtualenv will use + Distribute by default (so you don't have to remember to use + ``--distribute``). + + +1.4.3 +----- + +* Include pip 0.6.1 + + +1.4.2 +----- + +* Fix pip installation on Windows + +* Fix use of stand-alone ``virtualenv.py`` (and boot scripts) + +* Exclude ~/.local (user site-packages) from environments when using + ``--no-site-packages`` + + +1.4.1 +----- + +* Include pip 0.6 + + +1.4 +--- + +* Updated setuptools to 0.6c11 + +* Added the --distribute option + +* Fixed packaging problem of support-files + + +1.3.4 +----- + +* Virtualenv now copies the actual embedded Python binary on + Mac OS X to fix a hang on Snow Leopard (10.6). + +* Fail more gracefully on Windows when ``win32api`` is not installed. + +* Fix site-packages taking precedent over Jython's ``__classpath__`` + and also specially handle the new ``__pyclasspath__`` entry in + ``sys.path``. + +* Now copies Jython's ``registry`` file to the virtualenv if it exists. + +* Better find libraries when compiling extensions on Windows. + +* Create ``Scripts\pythonw.exe`` on Windows. + +* Added support for the Debian/Ubuntu + ``/usr/lib/pythonX.Y/dist-packages`` directory. + +* Set ``distutils.sysconfig.get_config_vars()['LIBDIR']`` (based on + ``sys.real_prefix``) which is reported to help building on Windows. + +* Make ``deactivate`` work on ksh + +* Fixes for ``--python``: make it work with ``--relocatable`` and the + symlink created to the exact Python version. + + +1.3.3 +----- + +* Use Windows newlines in ``activate.bat``, which has been reported to help + when using non-ASCII directory names. + +* Fixed compatibility with Jython 2.5b1. + +* Added a function ``virtualenv.install_python`` for more fine-grained + access to what ``virtualenv.create_environment`` does. + +* Fix `a problem <https://bugs.launchpad.net/virtualenv/+bug/241581>`_ + with Windows and paths that contain spaces. + +* If ``/path/to/env/.pydistutils.cfg`` exists (or + ``/path/to/env/pydistutils.cfg`` on Windows systems) then ignore + ``~/.pydistutils.cfg`` and use that other file instead. + +* Fix ` a problem + <https://bugs.launchpad.net/virtualenv/+bug/340050>`_ picking up + some ``.so`` libraries in ``/usr/local``. + + +1.3.2 +----- + +* Remove the ``[install] prefix = ...`` setting from the virtualenv + ``distutils.cfg`` -- this has been causing problems for a lot of + people, in rather obscure ways. + +* If you use a boot script it will attempt to import ``virtualenv`` + and find a pre-downloaded Setuptools egg using that. + +* Added platform-specific paths, like ``/usr/lib/pythonX.Y/plat-linux2`` + + +1.3.1 +----- + +* Real Python 2.6 compatibility. Backported the Python 2.6 updates to + ``site.py``, including `user directories + <http://docs.python.org/dev/whatsnew/2.6.html#pep-370-per-user-site-packages-directory>`_ + (this means older versions of Python will support user directories, + whether intended or not). + +* Always set ``[install] prefix`` in ``distutils.cfg`` -- previously + on some platforms where a system-wide ``distutils.cfg`` was present + with a ``prefix`` setting, packages would be installed globally + (usually in ``/usr/local/lib/pythonX.Y/site-packages``). + +* Sometimes Cygwin seems to leave ``.exe`` off ``sys.executable``; a + workaround is added. + +* Fix ``--python`` option. + +* Fixed handling of Jython environments that use a + jython-complete.jar. + + +1.3 +--- + +* Update to Setuptools 0.6c9 +* Added an option ``virtualenv --relocatable EXISTING_ENV``, which + will make an existing environment "relocatable" -- the paths will + not be absolute in scripts, ``.egg-info`` and ``.pth`` files. This + may assist in building environments that can be moved and copied. + You have to run this *after* any new packages installed. +* Added ``bin/activate_this.py``, a file you can use like + ``execfile("path_to/activate_this.py", + dict(__file__="path_to/activate_this.py"))`` -- this will activate + the environment in place, similar to what `the mod_wsgi example + does <http://code.google.com/p/modwsgi/wiki/VirtualEnvironments>`_. +* For Mac framework builds of Python, the site-packages directory + ``/Library/Python/X.Y/site-packages`` is added to ``sys.path``, from + Andrea Rech. +* Some platform-specific modules in Macs are added to the path now + (``plat-darwin/``, ``plat-mac/``, ``plat-mac/lib-scriptpackages``), + from Andrea Rech. +* Fixed a small Bashism in the ``bin/activate`` shell script. +* Added ``__future__`` to the list of required modules, for Python + 2.3. You'll still need to backport your own ``subprocess`` module. +* Fixed the ``__classpath__`` entry in Jython's ``sys.path`` taking + precedent over virtualenv's libs. + + +1.2 +--- + +* Added a ``--python`` option to select the Python interpreter. +* Add ``warnings`` to the modules copied over, for Python 2.6 support. +* Add ``sets`` to the module copied over for Python 2.3 (though Python + 2.3 still probably doesn't work). + + +1.1.1 +----- + +* Added support for Jython 2.5. + + +1.1 +--- + +* Added support for Python 2.6. +* Fix a problem with missing ``DLLs/zlib.pyd`` on Windows. Create +* ``bin/python`` (or ``bin/python.exe``) even when you run virtualenv + with an interpreter named, e.g., ``python2.4`` +* Fix MacPorts Python +* Added --unzip-setuptools option +* Update to Setuptools 0.6c8 +* If the current directory is not writable, run ez_setup.py in ``/tmp`` +* Copy or symlink over the ``include`` directory so that packages will + more consistently compile. + + +1.0 +--- + +* Fix build on systems that use ``/usr/lib64``, distinct from + ``/usr/lib`` (specifically CentOS x64). +* Fixed bug in ``--clear``. +* Fixed typos in ``deactivate.bat``. +* Preserve ``$PYTHONPATH`` when calling subprocesses. + + +0.9.2 +----- + +* Fix include dir copying on Windows (makes compiling possible). +* Include the main ``lib-tk`` in the path. +* Patch ``distutils.sysconfig``: ``get_python_inc`` and + ``get_python_lib`` to point to the global locations. +* Install ``distutils.cfg`` before Setuptools, so that system + customizations of ``distutils.cfg`` won't effect the installation. +* Add ``bin/pythonX.Y`` to the virtualenv (in addition to + ``bin/python``). +* Fixed an issue with Mac Framework Python builds, and absolute paths + (from Ronald Oussoren). + + +0.9.1 +----- + +* Improve ability to create a virtualenv from inside a virtualenv. +* Fix a little bug in ``bin/activate``. +* Actually get ``distutils.cfg`` to work reliably. + + +0.9 +--- + +* Added ``lib-dynload`` and ``config`` to things that need to be + copied over in an environment. +* Copy over or symlink the ``include`` directory, so that you can + build packages that need the C headers. +* Include a ``distutils`` package, so you can locally update + ``distutils.cfg`` (in ``lib/pythonX.Y/distutils/distutils.cfg``). +* Better avoid downloading Setuptools, and hitting PyPI on environment + creation. +* Fix a problem creating a ``lib64/`` directory. +* Should work on MacOSX Framework builds (the default Python + installations on Mac). Thanks to Ronald Oussoren. + + +0.8.4 +----- + +* Windows installs would sometimes give errors about ``sys.prefix`` that + were inaccurate. +* Slightly prettier output. + + +0.8.3 +----- + +* Added support for Windows. + + +0.8.2 +----- + +* Give a better warning if you are on an unsupported platform (Mac + Framework Pythons, and Windows). +* Give error about running while inside a workingenv. +* Give better error message about Python 2.3. + + +0.8.1 +----- + +Fixed packaging of the library. + + +0.8 +--- + +Initial release. Everything is changed and new! diff --git a/testing/mozharness/external_tools/virtualenv/docs/conf.py b/testing/mozharness/external_tools/virtualenv/docs/conf.py new file mode 100644 index 000000000..9332aa1bc --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/conf.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# +# Paste documentation build configuration file, created by +# sphinx-quickstart on Tue Apr 22 22:08:49 2008. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# The contents of this file are pickled, so don't put values in the namespace +# that aren't pickleable (module imports are okay, they're removed automatically). +# +# All configuration values have a default value; values that are commented out +# serve to show the default value. + +import os +import sys + +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +# If your extensions are in another directory, add it here. +sys.path.insert(0, os.path.abspath(os.pardir)) + +# General configuration +# --------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.extlinks'] + +# Add any paths that contain templates here, relative to this directory. +#templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General substitutions. +project = 'virtualenv' +copyright = '2007-2014, Ian Bicking, The Open Planning Project, PyPA' + +# The default replacements for |version| and |release|, also used in various +# other places throughout the built documents. +try: + from virtualenv import __version__ + # The short X.Y version. + version = '.'.join(__version__.split('.')[:2]) + # The full version, including alpha/beta/rc tags. + release = __version__ +except ImportError: + version = release = 'dev' + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +unused_docs = [] + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +extlinks = { + 'issue': ('https://github.com/pypa/virtualenv/issues/%s', '#'), + 'pull': ('https://github.com/pypa/virtualenv/pull/%s', 'PR #'), +} + + +# Options for HTML output +# ----------------------- + +# The style sheet to use for HTML and HTML Help pages. A file of that name +# must exist either in Sphinx' static/ path, or in one of the custom paths +# given in html_static_path. +#html_style = 'default.css' + +html_theme = 'default' +if not on_rtd: + try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + except ImportError: + pass + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Content template for the index page. +#html_index = '' + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If true, the reST sources are included in the HTML build as _sources/<name>. +#html_copy_source = True + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Pastedoc' + + +# Options for LaTeX output +# ------------------------ + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, document class [howto/manual]). +#latex_documents = [] + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/testing/mozharness/external_tools/virtualenv/docs/development.rst b/testing/mozharness/external_tools/virtualenv/docs/development.rst new file mode 100644 index 000000000..aba2785a3 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/development.rst @@ -0,0 +1,61 @@ +Development +=========== + +Contributing +------------ + +Refer to the `pip development`_ documentation - it applies equally to +virtualenv, except that virtualenv issues should filed on the `virtualenv +repo`_ at GitHub. + +Virtualenv's release schedule is tied to pip's -- each time there's a new pip +release, there will be a new virtualenv release that bundles the new version of +pip. + +Files in the `virtualenv_embedded/` subdirectory are embedded into +`virtualenv.py` itself as base64-encoded strings (in order to support +single-file use of `virtualenv.py` without installing it). If your patch +changes any file in `virtualenv_embedded/`, run `bin/rebuild-script.py` to +update the embedded version of that file in `virtualenv.py`; commit that and +submit it as part of your patch / pull request. + +.. _pip development: http://www.pip-installer.org/en/latest/development.html +.. _virtualenv repo: https://github.com/pypa/virtualenv/ + +Running the tests +----------------- + +Virtualenv's test suite is small and not yet at all comprehensive, but we aim +to grow it. + +The easy way to run tests (handles test dependencies automatically):: + + $ python setup.py test + +If you want to run only a selection of the tests, you'll need to run them +directly with pytest instead. Create a virtualenv, and install required +packages:: + + $ pip install pytest mock + +Run pytest:: + + $ pytest + +Or select just a single test file to run:: + + $ pytest tests/test_virtualenv + +Status and License +------------------ + +``virtualenv`` is a successor to `workingenv +<http://cheeseshop.python.org/pypi/workingenv.py>`_, and an extension +of `virtual-python +<http://peak.telecommunity.com/DevCenter/EasyInstall#creating-a-virtual-python>`_. + +It was written by Ian Bicking, sponsored by the `Open Planning +Project <http://openplans.org>`_ and is now maintained by a +`group of developers <https://github.com/pypa/virtualenv/raw/master/AUTHORS.txt>`_. +It is licensed under an +`MIT-style permissive license <https://github.com/pypa/virtualenv/raw/master/LICENSE.txt>`_. diff --git a/testing/mozharness/external_tools/virtualenv/docs/index.rst b/testing/mozharness/external_tools/virtualenv/docs/index.rst new file mode 100644 index 000000000..e745a87b7 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/index.rst @@ -0,0 +1,137 @@ +Virtualenv +========== + +`Mailing list <http://groups.google.com/group/python-virtualenv>`_ | +`Issues <https://github.com/pypa/virtualenv/issues>`_ | +`Github <https://github.com/pypa/virtualenv>`_ | +`PyPI <https://pypi.python.org/pypi/virtualenv/>`_ | +User IRC: #pypa +Dev IRC: #pypa-dev + +Introduction +------------ + +``virtualenv`` is a tool to create isolated Python environments. + +The basic problem being addressed is one of dependencies and versions, +and indirectly permissions. Imagine you have an application that +needs version 1 of LibFoo, but another application requires version +2. How can you use both these applications? If you install +everything into ``/usr/lib/python2.7/site-packages`` (or whatever your +platform's standard location is), it's easy to end up in a situation +where you unintentionally upgrade an application that shouldn't be +upgraded. + +Or more generally, what if you want to install an application *and +leave it be*? If an application works, any change in its libraries or +the versions of those libraries can break the application. + +Also, what if you can't install packages into the global +``site-packages`` directory? For instance, on a shared host. + +In all these cases, ``virtualenv`` can help you. It creates an +environment that has its own installation directories, that doesn't +share libraries with other virtualenv environments (and optionally +doesn't access the globally installed libraries either). + +.. comment: split here + +.. toctree:: + :maxdepth: 2 + + installation + userguide + reference + development + changes + +.. warning:: + + Python bugfix releases 2.6.8, 2.7.3, 3.1.5 and 3.2.3 include a change that + will cause "import random" to fail with "cannot import name urandom" on any + virtualenv created on a Unix host with an earlier release of Python + 2.6/2.7/3.1/3.2, if the underlying system Python is upgraded. This is due to + the fact that a virtualenv uses the system Python's standard library but + contains its own copy of the Python interpreter, so an upgrade to the system + Python results in a mismatch between the version of the Python interpreter + and the version of the standard library. It can be fixed by removing + ``$ENV/bin/python`` and re-running virtualenv on the same target directory + with the upgraded Python. + +Other Documentation and Links +----------------------------- + +* `Blog announcement of virtualenv`__. + + .. __: http://blog.ianbicking.org/2007/10/10/workingenv-is-dead-long-live-virtualenv/ + +* James Gardner has written a tutorial on using `virtualenv with + Pylons + <http://wiki.pylonshq.com/display/pylonscookbook/Using+a+Virtualenv+Sandbox>`_. + +* Chris Perkins created a `showmedo video including virtualenv + <http://showmedo.com/videos/video?name=2910000&fromSeriesID=291>`_. + +* Doug Hellmann's `virtualenvwrapper`_ is a useful set of scripts to make + your workflow with many virtualenvs even easier. `His initial blog post on it`__. + He also wrote `an example of using virtualenv to try IPython`__. + + .. _virtualenvwrapper: https://pypi.python.org/pypi/virtualenvwrapper/ + .. __: https://doughellmann.com/blog/2008/05/01/virtualenvwrapper/ + .. __: https://doughellmann.com/blog/2008/02/01/ipython-and-virtualenv/ + +* `Pew`_ is another wrapper for virtualenv that makes use of a different + activation technique. + + .. _Pew: https://pypi.python.org/pypi/pew/ + +* `Using virtualenv with mod_wsgi + <http://code.google.com/p/modwsgi/wiki/VirtualEnvironments>`_. + +* `virtualenv commands + <https://github.com/thisismedium/virtualenv-commands>`_ for some more + workflow-related tools around virtualenv. + +* PyCon US 2011 talk: `Reverse-engineering Ian Bicking's brain: inside pip and virtualenv + <http://pyvideo.org/video/568/reverse-engineering-ian-bicking--39-s-brain--insi>`_. + By the end of the talk, you'll have a good idea exactly how pip + and virtualenv do their magic, and where to go looking in the source + for particular behaviors or bug fixes. + +Compare & Contrast with Alternatives +------------------------------------ + +There are several alternatives that create isolated environments: + +* ``workingenv`` (which I do not suggest you use anymore) is the + predecessor to this library. It used the main Python interpreter, + but relied on setting ``$PYTHONPATH`` to activate the environment. + This causes problems when running Python scripts that aren't part of + the environment (e.g., a globally installed ``hg`` or ``bzr``). It + also conflicted a lot with Setuptools. + +* `virtual-python + <http://peak.telecommunity.com/DevCenter/EasyInstall#creating-a-virtual-python>`_ + is also a predecessor to this library. It uses only symlinks, so it + couldn't work on Windows. It also symlinks over the *entire* + standard library and global ``site-packages``. As a result, it + won't see new additions to the global ``site-packages``. + + This script only symlinks a small portion of the standard library + into the environment, and so on Windows it is feasible to simply + copy these files over. Also, it creates a new/empty + ``site-packages`` and also adds the global ``site-packages`` to the + path, so updates are tracked separately. This script also installs + Setuptools automatically, saving a step and avoiding the need for + network access. + +* `zc.buildout <http://pypi.python.org/pypi/zc.buildout>`_ doesn't + create an isolated Python environment in the same style, but + achieves similar results through a declarative config file that sets + up scripts with very particular packages. As a declarative system, + it is somewhat easier to repeat and manage, but more difficult to + experiment with. ``zc.buildout`` includes the ability to setup + non-Python systems (e.g., a database server or an Apache instance). + +I *strongly* recommend anyone doing application development or +deployment use one of these tools. diff --git a/testing/mozharness/external_tools/virtualenv/docs/installation.rst b/testing/mozharness/external_tools/virtualenv/docs/installation.rst new file mode 100644 index 000000000..3006d7617 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/installation.rst @@ -0,0 +1,58 @@ +Installation +============ + +.. warning:: + + We advise installing virtualenv-1.9 or greater. Prior to version 1.9, the + pip included in virtualenv did not download from PyPI over SSL. + +.. warning:: + + When using pip to install virtualenv, we advise using pip 1.3 or greater. + Prior to version 1.3, pip did not download from PyPI over SSL. + +.. warning:: + + We advise against using easy_install to install virtualenv when using + setuptools < 0.9.7, because easy_install didn't download from PyPI over SSL + and was broken in some subtle ways. + +To install globally with `pip` (if you have pip 1.3 or greater installed globally): + +:: + + $ [sudo] pip install virtualenv + +Or to get the latest unreleased dev version: + +:: + + $ [sudo] pip install https://github.com/pypa/virtualenv/tarball/develop + + +To install version X.X globally from source: + +:: + + $ curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-X.X.tar.gz + $ tar xvfz virtualenv-X.X.tar.gz + $ cd virtualenv-X.X + $ [sudo] python setup.py install + + +To *use* locally from source: + +:: + + $ curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-X.X.tar.gz + $ tar xvfz virtualenv-X.X.tar.gz + $ cd virtualenv-X.X + $ python virtualenv.py myVE + +.. note:: + + The ``virtualenv.py`` script is *not* supported if run without the + necessary pip/setuptools/virtualenv distributions available locally. All + of the installation methods above include a ``virtualenv_support`` + directory alongside ``virtualenv.py`` which contains a complete set of + pip and setuptools distributions, and so are fully supported. diff --git a/testing/mozharness/external_tools/virtualenv/docs/make.bat b/testing/mozharness/external_tools/virtualenv/docs/make.bat new file mode 100644 index 000000000..aa5c189fc --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/make.bat @@ -0,0 +1,170 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^<target^>` where ^<target^> is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-compressor.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-compressor.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/testing/mozharness/external_tools/virtualenv/docs/reference.rst b/testing/mozharness/external_tools/virtualenv/docs/reference.rst new file mode 100644 index 000000000..9249473c9 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/reference.rst @@ -0,0 +1,261 @@ +Reference Guide +=============== + +``virtualenv`` Command +---------------------- + +.. _usage: + +Usage +~~~~~ + +:command:`virtualenv [OPTIONS] ENV_DIR` + + Where ``ENV_DIR`` is an absolute or relative path to a directory to create + the virtual environment in. + +.. _options: + +Options +~~~~~~~ + +.. program: virtualenv + +.. option:: --version + + show program's version number and exit + +.. option:: -h, --help + + show this help message and exit + +.. option:: -v, --verbose + + Increase verbosity. + +.. option:: -q, --quiet + + Decrease verbosity. + +.. option:: -p PYTHON_EXE, --python=PYTHON_EXE + + The Python interpreter to use, e.g., + --python=python2.5 will use the python2.5 interpreter + to create the new environment. The default is the + interpreter that virtualenv was installed with + (like ``/usr/bin/python``) + +.. option:: --clear + + Clear out the non-root install and start from scratch. + +.. option:: --system-site-packages + + Give the virtual environment access to the global + site-packages. + +.. option:: --always-copy + + Always copy files rather than symlinking. + +.. option:: --relocatable + + Make an EXISTING virtualenv environment relocatable. + This fixes up scripts and makes all .pth files relative. + +.. option:: --unzip-setuptools + + Unzip Setuptools when installing it. + +.. option:: --no-setuptools + + Do not install setuptools in the new virtualenv. + +.. option:: --no-pip + + Do not install pip in the new virtualenv. + +.. option:: --no-wheel + + Do not install wheel in the new virtualenv. + +.. option:: --extra-search-dir=DIR + + Directory to look for setuptools/pip distributions in. + This option can be specified multiple times. + +.. option:: --prompt=PROMPT + + Provides an alternative prompt prefix for this + environment. + +.. option:: --download + + Download preinstalled packages from PyPI. + +.. option:: --no-download + + Do not download preinstalled packages from PyPI. + +.. option:: --no-site-packages + + DEPRECATED. Retained only for backward compatibility. + Not having access to global site-packages is now the + default behavior. + +.. option:: --distribute +.. option:: --setuptools + + Legacy; now have no effect. Before version 1.10 these could be used + to choose whether to install Distribute_ or Setuptools_ into the created + virtualenv. Distribute has now been merged into Setuptools, and the + latter is always installed. + +.. _Distribute: https://pypi.python.org/pypi/distribute +.. _Setuptools: https://pypi.python.org/pypi/setuptools + + +Configuration +------------- + +Environment Variables +~~~~~~~~~~~~~~~~~~~~~ + +Each command line option is automatically used to look for environment +variables with the name format ``VIRTUALENV_<UPPER_NAME>``. That means +the name of the command line options are capitalized and have dashes +(``'-'``) replaced with underscores (``'_'``). + +For example, to automatically use a custom Python binary instead of the +one virtualenv is run with you can also set an environment variable:: + + $ export VIRTUALENV_PYTHON=/opt/python-3.3/bin/python + $ virtualenv ENV + +It's the same as passing the option to virtualenv directly:: + + $ virtualenv --python=/opt/python-3.3/bin/python ENV + +This also works for appending command line options, like ``--find-links``. +Just leave an empty space between the passed values, e.g.:: + + $ export VIRTUALENV_EXTRA_SEARCH_DIR="/path/to/dists /path/to/other/dists" + $ virtualenv ENV + +is the same as calling:: + + $ virtualenv --extra-search-dir=/path/to/dists --extra-search-dir=/path/to/other/dists ENV + +.. envvar:: VIRTUAL_ENV_DISABLE_PROMPT + + Any virtualenv created when this is set to a non-empty value will not have + it's :ref:`activate` modify the shell prompt. + + +Configuration File +~~~~~~~~~~~~~~~~~~ + +virtualenv also looks for a standard ini config file. On Unix and Mac OS X +that's ``$HOME/.virtualenv/virtualenv.ini`` and on Windows, it's +``%APPDATA%\virtualenv\virtualenv.ini``. + +The names of the settings are derived from the long command line option, +e.g. the option :option:`--python <-p>` would look like this:: + + [virtualenv] + python = /opt/python-3.3/bin/python + +Appending options like :option:`--extra-search-dir` can be written on multiple +lines:: + + [virtualenv] + extra-search-dir = + /path/to/dists + /path/to/other/dists + +Please have a look at the output of :option:`--help <-h>` for a full list +of supported options. + + +Extending Virtualenv +-------------------- + + +Creating Your Own Bootstrap Scripts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While this creates an environment, it doesn't put anything into the +environment. Developers may find it useful to distribute a script +that sets up a particular environment, for example a script that +installs a particular web application. + +To create a script like this, call +:py:func:`virtualenv.create_bootstrap_script`, and write the +result to your new bootstrapping script. + +.. py:function:: create_bootstrap_script(extra_text) + + Creates a bootstrap script from ``extra_text``, which is like + this script but with extend_parser, adjust_options, and after_install hooks. + +This returns a string that (written to disk of course) can be used +as a bootstrap script with your own customizations. The script +will be the standard virtualenv.py script, with your extra text +added (your extra text should be Python code). + +If you include these functions, they will be called: + +.. py:function:: extend_parser(optparse_parser) + + You can add or remove options from the parser here. + +.. py:function:: adjust_options(options, args) + + You can change options here, or change the args (if you accept + different kinds of arguments, be sure you modify ``args`` so it is + only ``[DEST_DIR]``). + +.. py:function:: after_install(options, home_dir) + + After everything is installed, this function is called. This + is probably the function you are most likely to use. An + example would be:: + + def after_install(options, home_dir): + if sys.platform == 'win32': + bin = 'Scripts' + else: + bin = 'bin' + subprocess.call([join(home_dir, bin, 'easy_install'), + 'MyPackage']) + subprocess.call([join(home_dir, bin, 'my-package-script'), + 'setup', home_dir]) + + This example immediately installs a package, and runs a setup + script from that package. + +Bootstrap Example +~~~~~~~~~~~~~~~~~ + +Here's a more concrete example of how you could use this:: + + import virtualenv, textwrap + output = virtualenv.create_bootstrap_script(textwrap.dedent(""" + import os, subprocess + def after_install(options, home_dir): + etc = join(home_dir, 'etc') + if not os.path.exists(etc): + os.makedirs(etc) + subprocess.call([join(home_dir, 'bin', 'easy_install'), + 'BlogApplication']) + subprocess.call([join(home_dir, 'bin', 'paster'), + 'make-config', 'BlogApplication', + join(etc, 'blog.ini')]) + subprocess.call([join(home_dir, 'bin', 'paster'), + 'setup-app', join(etc, 'blog.ini')]) + """)) + f = open('blog-bootstrap.py', 'w').write(output) + +Another example is available `here`__. + +.. __: https://github.com/socialplanning/fassembler/blob/master/fassembler/create-venv-script.py diff --git a/testing/mozharness/external_tools/virtualenv/docs/userguide.rst b/testing/mozharness/external_tools/virtualenv/docs/userguide.rst new file mode 100644 index 000000000..35f0dc950 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/docs/userguide.rst @@ -0,0 +1,258 @@ +User Guide +========== + + +Usage +----- + +Virtualenv has one basic command:: + + $ virtualenv ENV + +Where ``ENV`` is a directory to place the new virtual environment. It has +a number of usual effects (modifiable by many :ref:`options`): + + - :file:`ENV/lib/` and :file:`ENV/include/` are created, containing supporting + library files for a new virtualenv python. Packages installed in this + environment will live under :file:`ENV/lib/pythonX.X/site-packages/`. + + - :file:`ENV/bin` is created, where executables live - noticeably a new + :command:`python`. Thus running a script with ``#! /path/to/ENV/bin/python`` + would run that script under this virtualenv's python. + + - The crucial packages pip_ and setuptools_ are installed, which allow other + packages to be easily installed to the environment. This associated pip + can be run from :file:`ENV/bin/pip`. + +The python in your new virtualenv is effectively isolated from the python that +was used to create it. + +.. _pip: https://pypi.python.org/pypi/pip +.. _setuptools: https://pypi.python.org/pypi/setuptools + + +.. _activate: + +activate script +~~~~~~~~~~~~~~~ + +In a newly created virtualenv there will also be a :command:`activate` shell +script. For Windows systems, activation scripts are provided for +the Command Prompt and Powershell. + +On Posix systems, this resides in :file:`/ENV/bin/`, so you can run:: + + $ source bin/activate + +For some shells (e.g. the original Bourne Shell) you may need to use the +:command:`.` command, when :command:`source` does not exist. There are also +separate activate files for some other shells, like csh and fish. +:file:`bin/activate` should work for bash/zsh/dash. + +This will change your ``$PATH`` so its first entry is the virtualenv's +``bin/`` directory. (You have to use ``source`` because it changes your +shell environment in-place.) This is all it does; it's purely a +convenience. If you directly run a script or the python interpreter +from the virtualenv's ``bin/`` directory (e.g. ``path/to/ENV/bin/pip`` +or ``/path/to/ENV/bin/python-script.py``) there's no need for +activation. + +The ``activate`` script will also modify your shell prompt to indicate +which environment is currently active. To disable this behaviour, see +:envvar:`VIRTUAL_ENV_DISABLE_PROMPT`. + +To undo these changes to your path (and prompt), just run:: + + $ deactivate + +On Windows, the equivalent `activate` script is in the ``Scripts`` folder:: + + > \path\to\env\Scripts\activate + +And type ``deactivate`` to undo the changes. + +Based on your active shell (CMD.exe or Powershell.exe), Windows will use +either activate.bat or activate.ps1 (as appropriate) to activate the +virtual environment. If using Powershell, see the notes about code signing +below. + +.. note:: + + If using Powershell, the ``activate`` script is subject to the + `execution policies`_ on the system. By default on Windows 7, the system's + excution policy is set to ``Restricted``, meaning no scripts like the + ``activate`` script are allowed to be executed. But that can't stop us + from changing that slightly to allow it to be executed. + + In order to use the script, you can relax your system's execution + policy to ``AllSigned``, meaning all scripts on the system must be + digitally signed to be executed. Since the virtualenv activation + script is signed by one of the authors (Jannis Leidel) this level of + the execution policy suffices. As an administrator run:: + + PS C:\> Set-ExecutionPolicy AllSigned + + Then you'll be asked to trust the signer, when executing the script. + You will be prompted with the following:: + + PS C:\> virtualenv .\foo + New python executable in C:\foo\Scripts\python.exe + Installing setuptools................done. + Installing pip...................done. + PS C:\> .\foo\scripts\activate + + Do you want to run software from this untrusted publisher? + File C:\foo\scripts\activate.ps1 is published by E=jannis@leidel.info, + CN=Jannis Leidel, L=Berlin, S=Berlin, C=DE, Description=581796-Gh7xfJxkxQSIO4E0 + and is not trusted on your system. Only run scripts from trusted publishers. + [V] Never run [D] Do not run [R] Run once [A] Always run [?] Help + (default is "D"):A + (foo) PS C:\> + + If you select ``[A] Always Run``, the certificate will be added to the + Trusted Publishers of your user account, and will be trusted in this + user's context henceforth. If you select ``[R] Run Once``, the script will + be run, but you will be prometed on a subsequent invocation. Advanced users + can add the signer's certificate to the Trusted Publishers of the Computer + account to apply to all users (though this technique is out of scope of this + document). + + Alternatively, you may relax the system execution policy to allow running + of local scripts without verifying the code signature using the following:: + + PS C:\> Set-ExecutionPolicy RemoteSigned + + Since the ``activate.ps1`` script is generated locally for each virtualenv, + it is not considered a remote script and can then be executed. + +.. _`execution policies`: http://technet.microsoft.com/en-us/library/dd347641.aspx + +Removing an Environment +~~~~~~~~~~~~~~~~~~~~~~~ + +Removing a virtual environment is simply done by deactivating it and deleting the +environment folder with all its contents:: + + (ENV)$ deactivate + $ rm -r /path/to/ENV + +The :option:`--system-site-packages` Option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you build with ``virtualenv --system-site-packages ENV``, your virtual +environment will inherit packages from ``/usr/lib/python2.7/site-packages`` +(or wherever your global site-packages directory is). + +This can be used if you have control over the global site-packages directory, +and you want to depend on the packages there. If you want isolation from the +global system, do not use this flag. + +Windows Notes +~~~~~~~~~~~~~ + +Some paths within the virtualenv are slightly different on Windows: scripts and +executables on Windows go in ``ENV\Scripts\`` instead of ``ENV/bin/`` and +libraries go in ``ENV\Lib\`` rather than ``ENV/lib/``. + +To create a virtualenv under a path with spaces in it on Windows, you'll need +the `win32api <http://sourceforge.net/projects/pywin32/>`_ library installed. + + +Using Virtualenv without ``bin/python`` +--------------------------------------- + +Sometimes you can't or don't want to use the Python interpreter +created by the virtualenv. For instance, in a `mod_python +<http://www.modpython.org/>`_ or `mod_wsgi <http://www.modwsgi.org/>`_ +environment, there is only one interpreter. + +Luckily, it's easy. You must use the custom Python interpreter to +*install* libraries. But to *use* libraries, you just have to be sure +the path is correct. A script is available to correct the path. You +can setup the environment like:: + + activate_this = '/path/to/env/bin/activate_this.py' + execfile(activate_this, dict(__file__=activate_this)) + +This will change ``sys.path`` and even change ``sys.prefix``, but also allow +you to use an existing interpreter. Items in your environment will show up +first on ``sys.path``, before global items. However, global items will +always be accessible (as if the :option:`--system-site-packages` flag had been +used in creating the environment, whether it was or not). Also, this cannot undo +the activation of other environments, or modules that have been imported. +You shouldn't try to, for instance, activate an environment before a web +request; you should activate *one* environment as early as possible, and not +do it again in that process. + +Making Environments Relocatable +------------------------------- + +**Note:** this option is somewhat experimental, and there are probably +caveats that have not yet been identified. + +.. warning:: + + The ``--relocatable`` option currently has a number of issues, + and is not guaranteed to work in all circumstances. It is possible + that the option will be deprecated in a future version of ``virtualenv``. + +Normally environments are tied to a specific path. That means that +you cannot move an environment around or copy it to another computer. +You can fix up an environment to make it relocatable with the +command:: + + $ virtualenv --relocatable ENV + +This will make some of the files created by setuptools use relative paths, +and will change all the scripts to use ``activate_this.py`` instead of using +the location of the Python interpreter to select the environment. + +**Note:** scripts which have been made relocatable will only work if +the virtualenv is activated, specifically the python executable from +the virtualenv must be the first one on the system PATH. Also note that +the activate scripts are not currently made relocatable by +``virtualenv --relocatable``. + +**Note:** you must run this after you've installed *any* packages into +the environment. If you make an environment relocatable, then +install a new package, you must run ``virtualenv --relocatable`` +again. + +Also, this **does not make your packages cross-platform**. You can +move the directory around, but it can only be used on other similar +computers. Some known environmental differences that can cause +incompatibilities: a different version of Python, when one platform +uses UCS2 for its internal unicode representation and another uses +UCS4 (a compile-time option), obvious platform changes like Windows +vs. Linux, or Intel vs. ARM, and if you have libraries that bind to C +libraries on the system, if those C libraries are located somewhere +different (either different versions, or a different filesystem +layout). + +If you use this flag to create an environment, currently, the +:option:`--system-site-packages` option will be implied. + +The :option:`--extra-search-dir` option +--------------------------------------- + +This option allows you to provide your own versions of setuptools and/or +pip to use instead of the embedded versions that come with virtualenv. + +To use this feature, pass one or more ``--extra-search-dir`` options to +virtualenv like this:: + + $ virtualenv --extra-search-dir=/path/to/distributions ENV + +The ``/path/to/distributions`` path should point to a directory that contains +setuptools and/or pip wheels. + +virtualenv will look for wheels in the specified directories, but will use +pip's standard algorithm for selecting the wheel to install, which looks for +the latest compatible wheel. + +As well as the extra directories, the search order includes: + +#. The ``virtualenv_support`` directory relative to virtualenv.py +#. The directory where virtualenv.py is located. +#. The current directory. + diff --git a/testing/mozharness/external_tools/virtualenv/scripts/virtualenv b/testing/mozharness/external_tools/virtualenv/scripts/virtualenv new file mode 100644 index 000000000..c961dd7db --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/scripts/virtualenv @@ -0,0 +1,3 @@ +#!/usr/bin/env python +import virtualenv +virtualenv.main() diff --git a/testing/mozharness/external_tools/virtualenv/setup.cfg b/testing/mozharness/external_tools/virtualenv/setup.cfg new file mode 100644 index 000000000..6662fa569 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/setup.cfg @@ -0,0 +1,8 @@ +[bdist_wheel] +universal = 1 + +[egg_info] +tag_date = 0 +tag_build = +tag_svn_revision = 0 + diff --git a/testing/mozharness/external_tools/virtualenv/setup.py b/testing/mozharness/external_tools/virtualenv/setup.py new file mode 100644 index 000000000..ee03bc531 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/setup.py @@ -0,0 +1,123 @@ +import os +import re +import shutil +import sys + +if sys.version_info[:2] < (2, 6): + sys.exit('virtualenv requires Python 2.6 or higher.') + +try: + from setuptools import setup + from setuptools.command.test import test as TestCommand + + class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = [] + + def finalize_options(self): + TestCommand.finalize_options(self) + #self.test_args = [] + #self.test_suite = True + + def run_tests(self): + # import here, because outside the eggs aren't loaded + import pytest + sys.exit(pytest.main(self.pytest_args)) + + setup_params = { + 'entry_points': { + 'console_scripts': ['virtualenv=virtualenv:main'], + }, + 'zip_safe': False, + 'cmdclass': {'test': PyTest}, + 'tests_require': ['pytest', 'mock'], + } +except ImportError: + from distutils.core import setup + if sys.platform == 'win32': + print('Note: without Setuptools installed you will ' + 'have to use "python -m virtualenv ENV"') + setup_params = {} + else: + script = 'scripts/virtualenv' + setup_params = {'scripts': [script]} + + +def read_file(*paths): + here = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(here, *paths)) as f: + return f.read() + +# Get long_description from index.rst: +long_description = read_file('docs', 'index.rst') +long_description = long_description.strip().split('split here', 1)[0] +# Add release history +changes = read_file('docs', 'changes.rst') +# Only report last two releases for brevity +releases_found = 0 +change_lines = [] +for line in changes.splitlines(): + change_lines.append(line) + if line.startswith('--------------'): + releases_found += 1 + if releases_found > 2: + break + +changes = '\n'.join(change_lines[:-2]) + '\n' +changes += '`Full Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_.' +# Replace issue/pull directives +changes = re.sub(r':pull:`(\d+)`', r'PR #\1', changes) +changes = re.sub(r':issue:`(\d+)`', r'#\1', changes) + +long_description += '\n\n' + changes + + +def get_version(): + version_file = read_file('virtualenv.py') + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + +# Hack to prevent stupid TypeError: 'NoneType' object is not callable error on +# exit of python setup.py test # in multiprocessing/util.py _exit_function when +# running python setup.py test (see +# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html) +try: + import multiprocessing # noqa +except ImportError: + pass + +setup( + name='virtualenv', + version=get_version(), + description="Virtual Python Environment builder", + long_description=long_description, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + keywords='setuptools deployment installation distutils', + author='Ian Bicking', + author_email='ianb@colorstudy.com', + maintainer='Jannis Leidel, Carl Meyer and Brian Rosner', + maintainer_email='python-virtualenv@groups.google.com', + url='https://virtualenv.pypa.io/', + license='MIT', + py_modules=['virtualenv'], + packages=['virtualenv_support'], + package_data={'virtualenv_support': ['*.whl']}, + **setup_params) diff --git a/testing/mozharness/external_tools/virtualenv/site.py b/testing/mozharness/external_tools/virtualenv/site.py new file mode 100644 index 000000000..4e426cdb6 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/site.py @@ -0,0 +1,760 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +In earlier versions of Python (up to 1.5a3), scripts or modules that +needed to use site-specific modules would place ``import site'' +somewhere near the top of their code. Because of the automatic +import, this is no longer necessary (but code that does it still +works). + +This will append site-specific paths to the module search path. On +Unix, it starts with sys.prefix and sys.exec_prefix (if different) and +appends lib/python<version>/site-packages as well as lib/site-python. +It also supports the Debian convention of +lib/python<version>/dist-packages. On other platforms (mainly Mac and +Windows), it uses just sys.prefix (and sys.exec_prefix, if different, +but this is unlikely). The resulting directories, if they exist, are +appended to sys.path, and also inspected for path configuration files. + +FOR DEBIAN, this sys.path is augmented with directories in /usr/local. +Local addons go into /usr/local/lib/python<version>/site-packages +(resp. /usr/local/lib/site-python), Debian addons install into +/usr/{lib,share}/python<version>/dist-packages. + +A path configuration file is a file whose name has the form +<package>.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.X/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.X/site-packages/bar + /usr/local/lib/python2.X/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +After these path manipulations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. + +""" + +import sys +import os + +try: + import __builtin__ as builtins +except ImportError: + import builtins +try: + set +except NameError: + from sets import Set as set + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = None +# for distutils.commands.install +USER_SITE = None +USER_BASE = None + +_is_64bit = (getattr(sys, 'maxsize', None) or getattr(sys, 'maxint')) > 2**32 +_is_pypy = hasattr(sys, 'pypy_version_info') +_is_jython = sys.platform[:4] == 'java' +if _is_jython: + ModuleType = type(os) + +def makepath(*paths): + dir = os.path.join(*paths) + if _is_jython and (dir == '__classpath__' or + dir.startswith('__pyclasspath__')): + return dir, dir + dir = os.path.abspath(dir) + return dir, os.path.normcase(dir) + +def abs__file__(): + """Set all module' __file__ attribute to an absolute path""" + for m in sys.modules.values(): + if ((_is_jython and not isinstance(m, ModuleType)) or + hasattr(m, '__loader__')): + # only modules need the abspath in Jython. and don't mess + # with a PEP 302-supplied __file__ + continue + f = getattr(m, '__file__', None) + if f is None: + continue + m.__file__ = os.path.abspath(f) + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if not dircase in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + +# XXX This should not be part of site.py, since it is needed even when +# using the -S option for Python. See http://www.python.org/sf/586680 +def addbuilddir(): + """Append ./build/lib.<platform> in case we're running in the build dir + (especially for Guido :-)""" + from distutils.util import get_platform + s = "build/lib.%s-%.3s" % (get_platform(), sys.version) + if hasattr(sys, 'gettotalrefcount'): + s += '-pydebug' + s = os.path.join(os.path.dirname(sys.path[-1]), s) + sys.path.append(s) + +def _init_pathinfo(): + """Return a set containing all existing directory entries from sys.path""" + d = set() + for dir in sys.path: + try: + if os.path.isdir(dir): + dir, dircase = makepath(dir) + d.add(dircase) + except TypeError: + continue + return d + +def addpackage(sitedir, name, known_paths): + """Add a new path to known_paths by combining sitedir and 'name' or execute + sitedir if it starts with 'import'""" + if known_paths is None: + _init_pathinfo() + reset = 1 + else: + reset = 0 + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "rU") + except IOError: + return + try: + for line in f: + if line.startswith("#"): + continue + if line.startswith("import"): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + finally: + f.close() + if reset: + known_paths = None + return known_paths + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = 1 + else: + reset = 0 + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + try: + names = os.listdir(sitedir) + except os.error: + return + names.sort() + for name in names: + if name.endswith(os.extsep + "pth"): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + +def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): + """Add site-packages (and possibly site-python) to sys.path""" + prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] + if exec_prefix != sys_prefix: + prefixes.append(os.path.join(exec_prefix, "local")) + + for prefix in prefixes: + if prefix: + if sys.platform in ('os2emx', 'riscos') or _is_jython: + sitedirs = [os.path.join(prefix, "Lib", "site-packages")] + elif _is_pypy: + sitedirs = [os.path.join(prefix, 'site-packages')] + elif sys.platform == 'darwin' and prefix == sys_prefix: + + if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python + + sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), + os.path.join(prefix, "Extras", "lib", "python")] + + else: # any other Python distros on OSX work this way + sitedirs = [os.path.join(prefix, "lib", + "python" + sys.version[:3], "site-packages")] + + elif os.sep == '/': + sitedirs = [os.path.join(prefix, + "lib", + "python" + sys.version[:3], + "site-packages"), + os.path.join(prefix, "lib", "site-python"), + os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] + lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") + if (os.path.exists(lib64_dir) and + os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): + if _is_64bit: + sitedirs.insert(0, lib64_dir) + else: + sitedirs.append(lib64_dir) + try: + # sys.getobjects only available in --with-pydebug build + sys.getobjects + sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) + except AttributeError: + pass + # Debian-specific dist-packages directories: + sitedirs.append(os.path.join(prefix, "local/lib", + "python" + sys.version[:3], + "dist-packages")) + if sys.version[0] == '2': + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[:3], + "dist-packages")) + else: + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[0], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "lib", "dist-python")) + else: + sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] + if sys.platform == 'darwin': + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' in prefix: + home = os.environ.get('HOME') + if home: + sitedirs.append( + os.path.join(home, + 'Library', + 'Python', + sys.version[:3], + 'site-packages')) + for sitedir in sitedirs: + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + return None + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + + USER_BASE is the root directory for all Python versions + + USER_SITE is the user specific site-packages directory + + USER_SITE/.. can be used for data. + """ + global USER_BASE, USER_SITE, ENABLE_USER_SITE + env_base = os.environ.get("PYTHONUSERBASE", None) + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + #if sys.platform in ('os2emx', 'riscos'): + # # Don't know what to put here + # USER_BASE = '' + # USER_SITE = '' + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser(base, "Python") + USER_SITE = os.path.join(USER_BASE, + "Python" + sys.version[0] + sys.version[2], + "site-packages") + else: + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser("~", ".local") + USER_SITE = os.path.join(USER_BASE, "lib", + "python" + sys.version[:3], + "site-packages") + + if ENABLE_USER_SITE and os.path.isdir(USER_SITE): + addsitedir(USER_SITE, known_paths) + if ENABLE_USER_SITE: + for dist_libdir in ("lib", "local/lib"): + user_site = os.path.join(USER_BASE, dist_libdir, + "python" + sys.version[:3], + "dist-packages") + if os.path.isdir(user_site): + addsitedir(user_site, known_paths) + return known_paths + + + +def setBEGINLIBPATH(): + """The OS/2 EMX port has optional extension modules that do double duty + as DLLs (and must use the .DLL file extension) for other extensions. + The library search path needs to be amended so these will be found + during module import. Use BEGINLIBPATH so that these are at the start + of the library search path. + + """ + dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") + libpath = os.environ['BEGINLIBPATH'].split(';') + if libpath[-1]: + libpath.append(dllpath) + else: + libpath[-1] = dllpath + os.environ['BEGINLIBPATH'] = ';'.join(libpath) + + +def setquit(): + """Define new built-ins 'quit' and 'exit'. + These are simply strings that display a hint on how to exit. + + """ + if os.sep == ':': + eof = 'Cmd-Q' + elif os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + class Quitter(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return 'Use %s() or %s to exit' % (self.name, eof) + def __call__(self, code=None): + # Shells like IDLE catch the SystemExit, but listen when their + # stdin wrapper is closed. + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + builtins.quit = Quitter('quit') + builtins.exit = Quitter('exit') + + +class _Printer(object): + """interactive prompt objects for printing the license text, a list of + contributors and the copyright notice.""" + + MAXLINES = 23 + + def __init__(self, name, data, files=(), dirs=()): + self.__name = name + self.__data = data + self.__files = files + self.__dirs = dirs + self.__lines = None + + def __setup(self): + if self.__lines: + return + data = None + for dir in self.__dirs: + for filename in self.__files: + filename = os.path.join(dir, filename) + try: + fp = open(filename, "rU") + data = fp.read() + fp.close() + break + except IOError: + pass + if data: + break + if not data: + data = self.__data + self.__lines = data.split('\n') + self.__linecnt = len(self.__lines) + + def __repr__(self): + self.__setup() + if len(self.__lines) <= self.MAXLINES: + return "\n".join(self.__lines) + else: + return "Type %s() to see the full %s text" % ((self.__name,)*2) + + def __call__(self): + self.__setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print(self.__lines[i]) + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + try: + key = raw_input(prompt) + except NameError: + key = input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break + +def setcopyright(): + """Set 'copyright' and 'credits' in __builtin__""" + builtins.copyright = _Printer("copyright", sys.copyright) + if _is_jython: + builtins.credits = _Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + elif _is_pypy: + builtins.credits = _Printer( + "credits", + "PyPy is maintained by the PyPy developers: http://pypy.org/") + else: + builtins.credits = _Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + here = os.path.dirname(os.__file__) + builtins.license = _Printer( + "license", "See http://www.python.org/%.3s/license.html" % sys.version, + ["LICENSE.txt", "LICENSE"], + [os.path.join(here, os.pardir), here, os.curdir]) + + +class _Helper(object): + """Define the built-in 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + +def sethelper(): + builtins.help = _Helper() + +def aliasmbcs(): + """On Windows, some default encodings are not provided by Python, + while they are always available as "mbcs" in each locale. Make + them usable by aliasing to "mbcs" in such a case.""" + if sys.platform == 'win32': + import locale, codecs + enc = locale.getdefaultlocale()[1] + if enc.startswith('cp'): # "cp***" ? + try: + codecs.lookup(enc) + except LookupError: + import encodings + encodings._cache[enc] = encodings._unknown + encodings.aliases.aliases[enc] = 'mbcs' + +def setencoding(): + """Set the string encoding used by the Unicode implementation. The + default is 'ascii', but if you're willing to experiment, you can + change this.""" + encoding = "ascii" # Default value set by _PyUnicode_Init() + if 0: + # Enable to support locale aware default string encodings. + import locale + loc = locale.getdefaultlocale() + if loc[1]: + encoding = loc[1] + if 0: + # Enable to switch off string to Unicode coercion and implicit + # Unicode to string conversion. + encoding = "undefined" + if encoding != "ascii": + # On Non-Unicode builds this will raise an AttributeError... + sys.setdefaultencoding(encoding) # Needs Python Unicode build ! + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + import sitecustomize + except ImportError: + pass + +def virtual_install_main_packages(): + f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) + sys.real_prefix = f.read().strip() + f.close() + pos = 2 + hardcoded_relative_dirs = [] + if sys.path[0] == '': + pos += 1 + if _is_jython: + paths = [os.path.join(sys.real_prefix, 'Lib')] + elif _is_pypy: + if sys.version_info > (3, 2): + cpyver = '%d' % sys.version_info[0] + elif sys.pypy_version_info >= (1, 5): + cpyver = '%d.%d' % sys.version_info[:2] + else: + cpyver = '%d.%d.%d' % sys.version_info[:3] + paths = [os.path.join(sys.real_prefix, 'lib_pypy'), + os.path.join(sys.real_prefix, 'lib-python', cpyver)] + if sys.pypy_version_info < (1, 9): + paths.insert(1, os.path.join(sys.real_prefix, + 'lib-python', 'modified-%s' % cpyver)) + hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below + # + # This is hardcoded in the Python executable, but relative to sys.prefix: + for path in paths[:]: + plat_path = os.path.join(path, 'plat-%s' % sys.platform) + if os.path.exists(plat_path): + paths.append(plat_path) + # MOZ: The MSYS2 and MinGW versions of Python have their main packages in the UNIX directory this checks specifically for the native win32 python + elif sys.platform == 'win32' and os.sep == '\\': + paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] + else: + paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] + hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below + lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) + if os.path.exists(lib64_path): + if _is_64bit: + paths.insert(0, lib64_path) + else: + paths.append(lib64_path) + # This is hardcoded in the Python executable, but relative to + # sys.prefix. Debian change: we need to add the multiarch triplet + # here, which is where the real stuff lives. As per PEP 421, in + # Python 3.3+, this lives in sys.implementation, while in Python 2.7 + # it lives in sys. + try: + arch = getattr(sys, 'implementation', sys)._multiarch + except AttributeError: + # This is a non-multiarch aware Python. Fallback to the old way. + arch = sys.platform + plat_path = os.path.join(sys.real_prefix, 'lib', + 'python'+sys.version[:3], + 'plat-%s' % arch) + if os.path.exists(plat_path): + paths.append(plat_path) + # This is hardcoded in the Python executable, but + # relative to sys.prefix, so we have to fix up: + for path in list(paths): + tk_dir = os.path.join(path, 'lib-tk') + if os.path.exists(tk_dir): + paths.append(tk_dir) + + # These are hardcoded in the Apple's Python executable, + # but relative to sys.prefix, so we have to fix them up: + if sys.platform == 'darwin': + hardcoded_paths = [os.path.join(relative_dir, module) + for relative_dir in hardcoded_relative_dirs + for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] + + for path in hardcoded_paths: + if os.path.exists(path): + paths.append(path) + + sys.path.extend(paths) + +def force_global_eggs_after_local_site_packages(): + """ + Force easy_installed eggs in the global environment to get placed + in sys.path after all packages inside the virtualenv. This + maintains the "least surprise" result that packages in the + virtualenv always mask global packages, never the other way + around. + + """ + egginsert = getattr(sys, '__egginsert', 0) + for i, path in enumerate(sys.path): + if i > egginsert and path.startswith(sys.prefix): + egginsert = i + sys.__egginsert = egginsert + 1 + +def virtual_addsitepackages(known_paths): + force_global_eggs_after_local_site_packages() + return addsitepackages(known_paths, sys_prefix=sys.real_prefix) + +def fixclasspath(): + """Adjust the special classpath sys.path entries for Jython. These + entries should follow the base virtualenv lib directories. + """ + paths = [] + classpaths = [] + for path in sys.path: + if path == '__classpath__' or path.startswith('__pyclasspath__'): + classpaths.append(path) + else: + paths.append(path) + sys.path = paths + sys.path.extend(classpaths) + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + import usercustomize + except ImportError: + pass + + +def main(): + global ENABLE_USER_SITE + virtual_install_main_packages() + abs__file__() + paths_in_sys = removeduppaths() + if (os.name == "posix" and sys.path and + os.path.basename(sys.path[-1]) == "Modules"): + addbuilddir() + if _is_jython: + fixclasspath() + GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) + if not GLOBAL_SITE_PACKAGES: + ENABLE_USER_SITE = False + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + paths_in_sys = addsitepackages(paths_in_sys) + paths_in_sys = addusersitepackages(paths_in_sys) + if GLOBAL_SITE_PACKAGES: + paths_in_sys = virtual_addsitepackages(paths_in_sys) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding + +main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + print("sys.path = [") + for dir in sys.path: + print(" %r," % (dir,)) + print("]") + def exists(path): + if os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + print("USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE))) + print("USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE))) + print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE) + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print(os.pathsep.join(buffer)) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/testing/mozharness/external_tools/virtualenv/tests/__init__.py b/testing/mozharness/external_tools/virtualenv/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/tests/__init__.py diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh b/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh new file mode 100755 index 000000000..e27727386 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +set -u + +ROOT="$(dirname $0)/.." +VIRTUALENV="${ROOT}/virtualenv.py" +TESTENV="/tmp/test_virtualenv_activate.venv" + +rm -rf ${TESTENV} + +echo "$0: Creating virtualenv ${TESTENV}..." 1>&2 + +${VIRTUALENV} ${TESTENV} | tee ${ROOT}/tests/test_activate_output.actual +if ! diff ${ROOT}/tests/test_activate_output.expected ${ROOT}/tests/test_activate_output.actual; then + echo "$0: Failed to get expected output from ${VIRTUALENV}!" 1>&2 + exit 1 +fi + +echo "$0: Created virtualenv ${TESTENV}." 1>&2 + +echo "$0: Activating ${TESTENV}..." 1>&2 +. ${TESTENV}/bin/activate +echo "$0: Activated ${TESTENV}." 1>&2 + +echo "$0: Checking value of \$VIRTUAL_ENV..." 1>&2 + +if [ "$VIRTUAL_ENV" != "${TESTENV}" ]; then + echo "$0: Expected \$VIRTUAL_ENV to be set to \"${TESTENV}\"; actual value: \"${VIRTUAL_ENV}\"!" 1>&2 + exit 2 +fi + +echo "$0: \$VIRTUAL_ENV = \"${VIRTUAL_ENV}\" -- OK." 1>&2 + +echo "$0: Checking output of \$(which python)..." 1>&2 + +if [ "$(which python)" != "${TESTENV}/bin/python" ]; then + echo "$0: Expected \$(which python) to return \"${TESTENV}/bin/python\"; actual value: \"$(which python)\"!" 1>&2 + exit 3 +fi + +echo "$0: Output of \$(which python) is OK." 1>&2 + +echo "$0: Checking output of \$(which pip)..." 1>&2 + +if [ "$(which pip)" != "${TESTENV}/bin/pip" ]; then + echo "$0: Expected \$(which pip) to return \"${TESTENV}/bin/pip\"; actual value: \"$(which pip)\"!" 1>&2 + exit 4 +fi + +echo "$0: Output of \$(which pip) is OK." 1>&2 + +echo "$0: Checking output of \$(which easy_install)..." 1>&2 + +if [ "$(which easy_install)" != "${TESTENV}/bin/easy_install" ]; then + echo "$0: Expected \$(which easy_install) to return \"${TESTENV}/bin/easy_install\"; actual value: \"$(which easy_install)\"!" 1>&2 + exit 5 +fi + +echo "$0: Output of \$(which easy_install) is OK." 1>&2 + +echo "$0: Executing a simple Python program..." 1>&2 + +TESTENV=${TESTENV} python <<__END__ +import os, sys + +expected_site_packages = os.path.join(os.environ['TESTENV'], 'lib','python%s' % sys.version[:3], 'site-packages') +site_packages = os.path.join(os.environ['VIRTUAL_ENV'], 'lib', 'python%s' % sys.version[:3], 'site-packages') + +assert site_packages == expected_site_packages, 'site_packages did not have expected value; actual value: %r' % site_packages + +open(os.path.join(site_packages, 'pydoc_test.py'), 'w').write('"""This is pydoc_test.py"""\n') +__END__ + +if [ $? -ne 0 ]; then + echo "$0: Python script failed!" 1>&2 + exit 6 +fi + +echo "$0: Execution of a simple Python program -- OK." 1>&2 + +echo "$0: Testing pydoc..." 1>&2 + +if ! PAGER=cat pydoc pydoc_test | grep 'This is pydoc_test.py' > /dev/null; then + echo "$0: pydoc test failed!" 1>&2 + exit 7 +fi + +echo "$0: pydoc is OK." 1>&2 + +echo "$0: Deactivating ${TESTENV}..." 1>&2 +deactivate +echo "$0: Deactivated ${TESTENV}." 1>&2 +echo "$0: OK!" 1>&2 + +rm -rf ${TESTENV} + diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected b/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected new file mode 100644 index 000000000..d49469feb --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected @@ -0,0 +1,2 @@ +New python executable in /tmp/test_virtualenv_activate.venv/bin/python +Installing setuptools, pip, wheel...done. diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py b/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py new file mode 100644 index 000000000..9682ef003 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py @@ -0,0 +1,44 @@ +import sys +import subprocess +import virtualenv +import pytest + +VIRTUALENV_SCRIPT = virtualenv.__file__ + +def test_commandline_basic(tmpdir): + """Simple command line usage should work""" + subprocess.check_call([ + sys.executable, + VIRTUALENV_SCRIPT, + str(tmpdir.join('venv')) + ]) + +def test_commandline_explicit_interp(tmpdir): + """Specifying the Python interpreter should work""" + subprocess.check_call([ + sys.executable, + VIRTUALENV_SCRIPT, + '-p', sys.executable, + str(tmpdir.join('venv')) + ]) + +# The registry lookups to support the abbreviated "-p 3.5" form of specifying +# a Python interpreter on Windows don't seem to work with Python 3.5. The +# registry layout is not well documented, and it's not clear that the feature +# is sufficiently widely used to be worth fixing. +# See https://github.com/pypa/virtualenv/issues/864 +@pytest.mark.skipif("sys.platform == 'win32' and sys.version_info[:2] >= (3,5)") +def test_commandline_abbrev_interp(tmpdir): + """Specifying abbreviated forms of the Python interpreter should work""" + if sys.platform == 'win32': + fmt = '%s.%s' + else: + fmt = 'python%s.%s' + abbrev = fmt % (sys.version_info[0], sys.version_info[1]) + subprocess.check_call([ + sys.executable, + VIRTUALENV_SCRIPT, + '-p', abbrev, + str(tmpdir.join('venv')) + ]) + diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py b/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py new file mode 100644 index 000000000..756cde936 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py @@ -0,0 +1,139 @@ +import virtualenv +import optparse +import os +import shutil +import sys +import tempfile +import pytest +import platform # noqa + +from mock import patch, Mock + + +def test_version(): + """Should have a version string""" + assert virtualenv.virtualenv_version, "Should have version" + + +@patch('os.path.exists') +def test_resolve_interpreter_with_absolute_path(mock_exists): + """Should return absolute path if given and exists""" + mock_exists.return_value = True + virtualenv.is_executable = Mock(return_value=True) + test_abs_path = os.path.abspath("/usr/bin/python53") + + exe = virtualenv.resolve_interpreter(test_abs_path) + + assert exe == test_abs_path, "Absolute path should return as is" + mock_exists.assert_called_with(test_abs_path) + virtualenv.is_executable.assert_called_with(test_abs_path) + + +@patch('os.path.exists') +def test_resolve_interpreter_with_nonexistent_interpreter(mock_exists): + """Should SystemExit with an nonexistent python interpreter path""" + mock_exists.return_value = False + + with pytest.raises(SystemExit): + virtualenv.resolve_interpreter("/usr/bin/python53") + + mock_exists.assert_called_with("/usr/bin/python53") + + +@patch('os.path.exists') +def test_resolve_interpreter_with_invalid_interpreter(mock_exists): + """Should exit when with absolute path if not exists""" + mock_exists.return_value = True + virtualenv.is_executable = Mock(return_value=False) + invalid = os.path.abspath("/usr/bin/pyt_hon53") + + with pytest.raises(SystemExit): + virtualenv.resolve_interpreter(invalid) + + mock_exists.assert_called_with(invalid) + virtualenv.is_executable.assert_called_with(invalid) + + +def test_activate_after_future_statements(): + """Should insert activation line after last future statement""" + script = [ + '#!/usr/bin/env python', + 'from __future__ import with_statement', + 'from __future__ import print_function', + 'print("Hello, world!")' + ] + assert virtualenv.relative_script(script) == [ + '#!/usr/bin/env python', + 'from __future__ import with_statement', + 'from __future__ import print_function', + '', + "import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del os, activate_this", + '', + 'print("Hello, world!")' + ] + + +def test_cop_update_defaults_with_store_false(): + """store_false options need reverted logic""" + class MyConfigOptionParser(virtualenv.ConfigOptionParser): + def __init__(self, *args, **kwargs): + self.config = virtualenv.ConfigParser.RawConfigParser() + self.files = [] + optparse.OptionParser.__init__(self, *args, **kwargs) + + def get_environ_vars(self, prefix='VIRTUALENV_'): + yield ("no_site_packages", "1") + + cop = MyConfigOptionParser() + cop.add_option( + '--no-site-packages', + dest='system_site_packages', + action='store_false', + help="Don't give access to the global site-packages dir to the " + "virtual environment (default)") + + defaults = {} + cop.update_defaults(defaults) + assert defaults == {'system_site_packages': 0} + +def test_install_python_bin(): + """Should create the right python executables and links""" + tmp_virtualenv = tempfile.mkdtemp() + try: + home_dir, lib_dir, inc_dir, bin_dir = \ + virtualenv.path_locations(tmp_virtualenv) + virtualenv.install_python(home_dir, lib_dir, inc_dir, bin_dir, False, + False) + + if virtualenv.is_win: + required_executables = [ 'python.exe', 'pythonw.exe'] + else: + py_exe_no_version = 'python' + py_exe_version_major = 'python%s' % sys.version_info[0] + py_exe_version_major_minor = 'python%s.%s' % ( + sys.version_info[0], sys.version_info[1]) + required_executables = [ py_exe_no_version, py_exe_version_major, + py_exe_version_major_minor ] + + for pth in required_executables: + assert os.path.exists(os.path.join(bin_dir, pth)), ("%s should " + "exist in bin_dir" % pth) + finally: + shutil.rmtree(tmp_virtualenv) + + +@pytest.mark.skipif("platform.python_implementation() == 'PyPy'") +def test_always_copy_option(): + """Should be no symlinks in directory tree""" + tmp_virtualenv = tempfile.mkdtemp() + ve_path = os.path.join(tmp_virtualenv, 'venv') + try: + virtualenv.create_environment(ve_path, symlink=False) + + for root, dirs, files in os.walk(tmp_virtualenv): + for f in files + dirs: + full_name = os.path.join(root, f) + assert not os.path.islink(full_name), "%s should not be a" \ + " symlink (to %s)" % (full_name, os.readlink(full_name)) + finally: + shutil.rmtree(tmp_virtualenv) diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv.py b/testing/mozharness/external_tools/virtualenv/virtualenv.py new file mode 100755 index 000000000..e363021cc --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv.py @@ -0,0 +1,2329 @@ +#!/usr/bin/env python +"""Create a "virtual" Python installation""" + +import os +import sys + +# If we are running in a new interpreter to create a virtualenv, +# we do NOT want paths from our existing location interfering with anything, +# So we remove this file's directory from sys.path - most likely to be +# the previous interpreter's site-packages. Solves #705, #763, #779 +if os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'): + for path in sys.path[:]: + if os.path.realpath(os.path.dirname(__file__)) == os.path.realpath(path): + sys.path.remove(path) + +import base64 +import codecs +import optparse +import re +import shutil +import logging +import zlib +import errno +import glob +import distutils.sysconfig +import struct +import subprocess +import pkgutil +import tempfile +import textwrap +from distutils.util import strtobool +from os.path import join + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +__version__ = "15.0.1" +virtualenv_version = __version__ # legacy + +if sys.version_info < (2, 6): + print('ERROR: %s' % sys.exc_info()[1]) + print('ERROR: this script requires Python 2.6 or greater.') + sys.exit(101) + +try: + basestring +except NameError: + basestring = str + +py_version = 'python%s.%s' % (sys.version_info[0], sys.version_info[1]) + +is_jython = sys.platform.startswith('java') +is_pypy = hasattr(sys, 'pypy_version_info') +is_win = (sys.platform == 'win32' and os.sep == '\\') +is_cygwin = (sys.platform == 'cygwin') +is_msys2 = (sys.platform == 'win32' and os.sep == '/') +is_darwin = (sys.platform == 'darwin') +abiflags = getattr(sys, 'abiflags', '') + +user_dir = os.path.expanduser('~') +if is_win: + default_storage_dir = os.path.join(user_dir, 'virtualenv') +else: + default_storage_dir = os.path.join(user_dir, '.virtualenv') +default_config_file = os.path.join(default_storage_dir, 'virtualenv.ini') + +if is_pypy: + expected_exe = 'pypy' +elif is_jython: + expected_exe = 'jython' +else: + expected_exe = 'python' + +# Return a mapping of version -> Python executable +# Only provided for Windows, where the information in the registry is used +if not is_win: + def get_installed_pythons(): + return {} +else: + try: + import winreg + except ImportError: + import _winreg as winreg + + def get_installed_pythons(): + try: + python_core = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, + "Software\\Python\\PythonCore") + except WindowsError: + # No registered Python installations + return {} + i = 0 + versions = [] + while True: + try: + versions.append(winreg.EnumKey(python_core, i)) + i = i + 1 + except WindowsError: + break + exes = dict() + for ver in versions: + try: + path = winreg.QueryValue(python_core, "%s\\InstallPath" % ver) + except WindowsError: + continue + exes[ver] = join(path, "python.exe") + + winreg.CloseKey(python_core) + + # Add the major versions + # Sort the keys, then repeatedly update the major version entry + # Last executable (i.e., highest version) wins with this approach + for ver in sorted(exes): + exes[ver[0]] = exes[ver] + + return exes + +REQUIRED_MODULES = ['os', 'posix', 'posixpath', 'nt', 'ntpath', 'genericpath', + 'fnmatch', 'locale', 'encodings', 'codecs', + 'stat', 'UserDict', 'readline', 'copy_reg', 'types', + 're', 'sre', 'sre_parse', 'sre_constants', 'sre_compile', + 'zlib'] + +REQUIRED_FILES = ['lib-dynload', 'config'] + +majver, minver = sys.version_info[:2] +if majver == 2: + if minver >= 6: + REQUIRED_MODULES.extend(['warnings', 'linecache', '_abcoll', 'abc']) + if minver >= 7: + REQUIRED_MODULES.extend(['_weakrefset']) + if is_msys2: + REQUIRED_MODULES.extend(['functools']) +elif majver == 3: + # Some extra modules are needed for Python 3, but different ones + # for different versions. + REQUIRED_MODULES.extend([ + '_abcoll', 'warnings', 'linecache', 'abc', 'io', '_weakrefset', + 'copyreg', 'tempfile', 'random', '__future__', 'collections', + 'keyword', 'tarfile', 'shutil', 'struct', 'copy', 'tokenize', + 'token', 'functools', 'heapq', 'bisect', 'weakref', 'reprlib' + ]) + if minver >= 2: + REQUIRED_FILES[-1] = 'config-%s' % majver + if minver >= 3: + import sysconfig + platdir = sysconfig.get_config_var('PLATDIR') + REQUIRED_FILES.append(platdir) + REQUIRED_MODULES.extend([ + 'base64', '_dummy_thread', 'hashlib', 'hmac', + 'imp', 'importlib', 'rlcompleter' + ]) + if minver >= 4: + REQUIRED_MODULES.extend([ + 'operator', + '_collections_abc', + '_bootlocale', + ]) + +if is_pypy: + # these are needed to correctly display the exceptions that may happen + # during the bootstrap + REQUIRED_MODULES.extend(['traceback', 'linecache']) + + +class Logger(object): + + """ + Logging object for use in command-line script. Allows ranges of + levels, to avoid some redundancy of displayed information. + """ + + DEBUG = logging.DEBUG + INFO = logging.INFO + NOTIFY = (logging.INFO+logging.WARN)/2 + WARN = WARNING = logging.WARN + ERROR = logging.ERROR + FATAL = logging.FATAL + + LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] + + def __init__(self, consumers): + self.consumers = consumers + self.indent = 0 + self.in_progress = None + self.in_progress_hanging = False + + def debug(self, msg, *args, **kw): + self.log(self.DEBUG, msg, *args, **kw) + + def info(self, msg, *args, **kw): + self.log(self.INFO, msg, *args, **kw) + + def notify(self, msg, *args, **kw): + self.log(self.NOTIFY, msg, *args, **kw) + + def warn(self, msg, *args, **kw): + self.log(self.WARN, msg, *args, **kw) + + def error(self, msg, *args, **kw): + self.log(self.ERROR, msg, *args, **kw) + + def fatal(self, msg, *args, **kw): + self.log(self.FATAL, msg, *args, **kw) + + def log(self, level, msg, *args, **kw): + if args: + if kw: + raise TypeError( + "You may give positional or keyword arguments, not both") + args = args or kw + rendered = None + for consumer_level, consumer in self.consumers: + if self.level_matches(level, consumer_level): + if (self.in_progress_hanging + and consumer in (sys.stdout, sys.stderr)): + self.in_progress_hanging = False + sys.stdout.write('\n') + sys.stdout.flush() + if rendered is None: + if args: + rendered = msg % args + else: + rendered = msg + rendered = ' '*self.indent + rendered + if hasattr(consumer, 'write'): + consumer.write(rendered+'\n') + else: + consumer(rendered) + + def start_progress(self, msg): + assert not self.in_progress, ( + "Tried to start_progress(%r) while in_progress %r" + % (msg, self.in_progress)) + if self.level_matches(self.NOTIFY, self._stdout_level()): + sys.stdout.write(msg) + sys.stdout.flush() + self.in_progress_hanging = True + else: + self.in_progress_hanging = False + self.in_progress = msg + + def end_progress(self, msg='done.'): + assert self.in_progress, ( + "Tried to end_progress without start_progress") + if self.stdout_level_matches(self.NOTIFY): + if not self.in_progress_hanging: + # Some message has been printed out since start_progress + sys.stdout.write('...' + self.in_progress + msg + '\n') + sys.stdout.flush() + else: + sys.stdout.write(msg + '\n') + sys.stdout.flush() + self.in_progress = None + self.in_progress_hanging = False + + def show_progress(self): + """If we are in a progress scope, and no log messages have been + shown, write out another '.'""" + if self.in_progress_hanging: + sys.stdout.write('.') + sys.stdout.flush() + + def stdout_level_matches(self, level): + """Returns true if a message at this level will go to stdout""" + return self.level_matches(level, self._stdout_level()) + + def _stdout_level(self): + """Returns the level that stdout runs at""" + for level, consumer in self.consumers: + if consumer is sys.stdout: + return level + return self.FATAL + + def level_matches(self, level, consumer_level): + """ + >>> l = Logger([]) + >>> l.level_matches(3, 4) + False + >>> l.level_matches(3, 2) + True + >>> l.level_matches(slice(None, 3), 3) + False + >>> l.level_matches(slice(None, 3), 2) + True + >>> l.level_matches(slice(1, 3), 1) + True + >>> l.level_matches(slice(2, 3), 1) + False + """ + if isinstance(level, slice): + start, stop = level.start, level.stop + if start is not None and start > consumer_level: + return False + if stop is not None and stop <= consumer_level: + return False + return True + else: + return level >= consumer_level + + #@classmethod + def level_for_integer(cls, level): + levels = cls.LEVELS + if level < 0: + return levels[0] + if level >= len(levels): + return levels[-1] + return levels[level] + + level_for_integer = classmethod(level_for_integer) + +# create a silent logger just to prevent this from being undefined +# will be overridden with requested verbosity main() is called. +logger = Logger([(Logger.LEVELS[-1], sys.stdout)]) + +def mkdir(path): + if not os.path.exists(path): + logger.info('Creating %s', path) + os.makedirs(path) + else: + logger.info('Directory %s already exists', path) + +def copyfileordir(src, dest, symlink=True): + if os.path.isdir(src): + shutil.copytree(src, dest, symlink) + else: + shutil.copy2(src, dest) + +def copyfile(src, dest, symlink=True): + if not os.path.exists(src): + # Some bad symlink in the src + logger.warn('Cannot find file %s (bad symlink)', src) + return + if os.path.exists(dest): + logger.debug('File %s already exists', dest) + return + if not os.path.exists(os.path.dirname(dest)): + logger.info('Creating parent directories for %s', os.path.dirname(dest)) + os.makedirs(os.path.dirname(dest)) + if not os.path.islink(src): + srcpath = os.path.abspath(src) + else: + srcpath = os.readlink(src) + if symlink and hasattr(os, 'symlink') and not is_win: + logger.info('Symlinking %s', dest) + try: + os.symlink(srcpath, dest) + except (OSError, NotImplementedError): + logger.info('Symlinking failed, copying to %s', dest) + copyfileordir(src, dest, symlink) + else: + logger.info('Copying to %s', dest) + copyfileordir(src, dest, symlink) + +def writefile(dest, content, overwrite=True): + if not os.path.exists(dest): + logger.info('Writing %s', dest) + with open(dest, 'wb') as f: + f.write(content.encode('utf-8')) + return + else: + with open(dest, 'rb') as f: + c = f.read() + if c != content.encode("utf-8"): + if not overwrite: + logger.notify('File %s exists with different content; not overwriting', dest) + return + logger.notify('Overwriting %s with new content', dest) + with open(dest, 'wb') as f: + f.write(content.encode('utf-8')) + else: + logger.info('Content %s already in place', dest) + +def rmtree(dir): + if os.path.exists(dir): + logger.notify('Deleting tree %s', dir) + shutil.rmtree(dir) + else: + logger.info('Do not need to delete %s; already gone', dir) + +def make_exe(fn): + if hasattr(os, 'chmod'): + oldmode = os.stat(fn).st_mode & 0xFFF # 0o7777 + newmode = (oldmode | 0x16D) & 0xFFF # 0o555, 0o7777 + os.chmod(fn, newmode) + logger.info('Changed mode of %s to %s', fn, oct(newmode)) + +def _find_file(filename, dirs): + for dir in reversed(dirs): + files = glob.glob(os.path.join(dir, filename)) + if files and os.path.isfile(files[0]): + return True, files[0] + return False, filename + +def file_search_dirs(): + here = os.path.dirname(os.path.abspath(__file__)) + dirs = [here, join(here, 'virtualenv_support')] + if os.path.splitext(os.path.dirname(__file__))[0] != 'virtualenv': + # Probably some boot script; just in case virtualenv is installed... + try: + import virtualenv + except ImportError: + pass + else: + dirs.append(os.path.join( + os.path.dirname(virtualenv.__file__), 'virtualenv_support')) + return [d for d in dirs if os.path.isdir(d)] + + +class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): + """ + Custom help formatter for use in ConfigOptionParser that updates + the defaults before expanding them, allowing them to show up correctly + in the help listing + """ + def expand_default(self, option): + if self.parser is not None: + self.parser.update_defaults(self.parser.defaults) + return optparse.IndentedHelpFormatter.expand_default(self, option) + + +class ConfigOptionParser(optparse.OptionParser): + """ + Custom option parser which updates its defaults by checking the + configuration files and environmental variables + """ + def __init__(self, *args, **kwargs): + self.config = ConfigParser.RawConfigParser() + self.files = self.get_config_files() + self.config.read(self.files) + optparse.OptionParser.__init__(self, *args, **kwargs) + + def get_config_files(self): + config_file = os.environ.get('VIRTUALENV_CONFIG_FILE', False) + if config_file and os.path.exists(config_file): + return [config_file] + return [default_config_file] + + def update_defaults(self, defaults): + """ + Updates the given defaults with values from the config files and + the environ. Does a little special handling for certain types of + options (lists). + """ + # Then go and look for the other sources of configuration: + config = {} + # 1. config files + config.update(dict(self.get_config_section('virtualenv'))) + # 2. environmental variables + config.update(dict(self.get_environ_vars())) + # Then set the options with those values + for key, val in config.items(): + key = key.replace('_', '-') + if not key.startswith('--'): + key = '--%s' % key # only prefer long opts + option = self.get_option(key) + if option is not None: + # ignore empty values + if not val: + continue + # handle multiline configs + if option.action == 'append': + val = val.split() + else: + option.nargs = 1 + if option.action == 'store_false': + val = not strtobool(val) + elif option.action in ('store_true', 'count'): + val = strtobool(val) + try: + val = option.convert_value(key, val) + except optparse.OptionValueError: + e = sys.exc_info()[1] + print("An error occurred during configuration: %s" % e) + sys.exit(3) + defaults[option.dest] = val + return defaults + + def get_config_section(self, name): + """ + Get a section of a configuration + """ + if self.config.has_section(name): + return self.config.items(name) + return [] + + def get_environ_vars(self, prefix='VIRTUALENV_'): + """ + Returns a generator with all environmental vars with prefix VIRTUALENV + """ + for key, val in os.environ.items(): + if key.startswith(prefix): + yield (key.replace(prefix, '').lower(), val) + + def get_default_values(self): + """ + Overridding to make updating the defaults after instantiation of + the option parser possible, update_defaults() does the dirty work. + """ + if not self.process_default_values: + # Old, pre-Optik 1.5 behaviour. + return optparse.Values(self.defaults) + + defaults = self.update_defaults(self.defaults.copy()) # ours + for option in self._get_all_options(): + default = defaults.get(option.dest) + if isinstance(default, basestring): + opt_str = option.get_opt_string() + defaults[option.dest] = option.check_value(opt_str, default) + return optparse.Values(defaults) + + +def main(): + parser = ConfigOptionParser( + version=virtualenv_version, + usage="%prog [OPTIONS] DEST_DIR", + formatter=UpdatingDefaultsHelpFormatter()) + + parser.add_option( + '-v', '--verbose', + action='count', + dest='verbose', + default=0, + help="Increase verbosity.") + + parser.add_option( + '-q', '--quiet', + action='count', + dest='quiet', + default=0, + help='Decrease verbosity.') + + parser.add_option( + '-p', '--python', + dest='python', + metavar='PYTHON_EXE', + help='The Python interpreter to use, e.g., --python=python2.5 will use the python2.5 ' + 'interpreter to create the new environment. The default is the interpreter that ' + 'virtualenv was installed with (%s)' % sys.executable) + + parser.add_option( + '--clear', + dest='clear', + action='store_true', + help="Clear out the non-root install and start from scratch.") + + parser.set_defaults(system_site_packages=False) + parser.add_option( + '--no-site-packages', + dest='system_site_packages', + action='store_false', + help="DEPRECATED. Retained only for backward compatibility. " + "Not having access to global site-packages is now the default behavior.") + + parser.add_option( + '--system-site-packages', + dest='system_site_packages', + action='store_true', + help="Give the virtual environment access to the global site-packages.") + + parser.add_option( + '--always-copy', + dest='symlink', + action='store_false', + default=True, + help="Always copy files rather than symlinking.") + + parser.add_option( + '--unzip-setuptools', + dest='unzip_setuptools', + action='store_true', + help="Unzip Setuptools when installing it.") + + parser.add_option( + '--relocatable', + dest='relocatable', + action='store_true', + help='Make an EXISTING virtualenv environment relocatable. ' + 'This fixes up scripts and makes all .pth files relative.') + + parser.add_option( + '--no-setuptools', + dest='no_setuptools', + action='store_true', + help='Do not install setuptools in the new virtualenv.') + + parser.add_option( + '--no-pip', + dest='no_pip', + action='store_true', + help='Do not install pip in the new virtualenv.') + + parser.add_option( + '--no-wheel', + dest='no_wheel', + action='store_true', + help='Do not install wheel in the new virtualenv.') + + default_search_dirs = file_search_dirs() + parser.add_option( + '--extra-search-dir', + dest="search_dirs", + action="append", + metavar='DIR', + default=default_search_dirs, + help="Directory to look for setuptools/pip distributions in. " + "This option can be used multiple times.") + + parser.add_option( + "--download", + dest="download", + default=True, + action="store_true", + help="Download preinstalled packages from PyPI.", + ) + + parser.add_option( + "--no-download", + '--never-download', + dest="download", + action="store_false", + help="Do not download preinstalled packages from PyPI.", + ) + + parser.add_option( + '--prompt', + dest='prompt', + help='Provides an alternative prompt prefix for this environment.') + + parser.add_option( + '--setuptools', + dest='setuptools', + action='store_true', + help="DEPRECATED. Retained only for backward compatibility. This option has no effect.") + + parser.add_option( + '--distribute', + dest='distribute', + action='store_true', + help="DEPRECATED. Retained only for backward compatibility. This option has no effect.") + + if 'extend_parser' in globals(): + extend_parser(parser) + + options, args = parser.parse_args() + + global logger + + if 'adjust_options' in globals(): + adjust_options(options, args) + + verbosity = options.verbose - options.quiet + logger = Logger([(Logger.level_for_integer(2 - verbosity), sys.stdout)]) + + if options.python and not os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'): + env = os.environ.copy() + interpreter = resolve_interpreter(options.python) + if interpreter == sys.executable: + logger.warn('Already using interpreter %s' % interpreter) + else: + logger.notify('Running virtualenv with interpreter %s' % interpreter) + env['VIRTUALENV_INTERPRETER_RUNNING'] = 'true' + file = __file__ + if file.endswith('.pyc'): + file = file[:-1] + popen = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env) + raise SystemExit(popen.wait()) + + if not args: + print('You must provide a DEST_DIR') + parser.print_help() + sys.exit(2) + if len(args) > 1: + print('There must be only one argument: DEST_DIR (you gave %s)' % ( + ' '.join(args))) + parser.print_help() + sys.exit(2) + + home_dir = args[0] + + if os.path.exists(home_dir) and os.path.isfile(home_dir): + logger.fatal('ERROR: File already exists and is not a directory.') + logger.fatal('Please provide a different path or delete the file.') + sys.exit(3) + + if os.environ.get('WORKING_ENV'): + logger.fatal('ERROR: you cannot run virtualenv while in a workingenv') + logger.fatal('Please deactivate your workingenv, then re-run this script') + sys.exit(3) + + if 'PYTHONHOME' in os.environ: + logger.warn('PYTHONHOME is set. You *must* activate the virtualenv before using it') + del os.environ['PYTHONHOME'] + + if options.relocatable: + make_environment_relocatable(home_dir) + return + + create_environment(home_dir, + site_packages=options.system_site_packages, + clear=options.clear, + unzip_setuptools=options.unzip_setuptools, + prompt=options.prompt, + search_dirs=options.search_dirs, + download=options.download, + no_setuptools=options.no_setuptools, + no_pip=options.no_pip, + no_wheel=options.no_wheel, + symlink=options.symlink and hasattr(os, 'symlink')) # MOZ: Make sure we don't use symlink when we don't have it + if 'after_install' in globals(): + after_install(options, home_dir) + +def call_subprocess(cmd, show_stdout=True, + filter_stdout=None, cwd=None, + raise_on_returncode=True, extra_env=None, + remove_from_env=None, stdin=None): + cmd_parts = [] + for part in cmd: + if len(part) > 45: + part = part[:20]+"..."+part[-20:] + if ' ' in part or '\n' in part or '"' in part or "'" in part: + part = '"%s"' % part.replace('"', '\\"') + if hasattr(part, 'decode'): + try: + part = part.decode(sys.getdefaultencoding()) + except UnicodeDecodeError: + part = part.decode(sys.getfilesystemencoding()) + cmd_parts.append(part) + cmd_desc = ' '.join(cmd_parts) + if show_stdout: + stdout = None + else: + stdout = subprocess.PIPE + logger.debug("Running command %s" % cmd_desc) + if extra_env or remove_from_env: + env = os.environ.copy() + if extra_env: + env.update(extra_env) + if remove_from_env: + for varname in remove_from_env: + env.pop(varname, None) + else: + env = None + try: + proc = subprocess.Popen( + cmd, stderr=subprocess.STDOUT, + stdin=None if stdin is None else subprocess.PIPE, + stdout=stdout, + cwd=cwd, env=env) + except Exception: + e = sys.exc_info()[1] + logger.fatal( + "Error %s while executing command %s" % (e, cmd_desc)) + raise + all_output = [] + if stdout is not None: + if stdin is not None: + proc.stdin.write(stdin) + proc.stdin.close() + + stdout = proc.stdout + encoding = sys.getdefaultencoding() + fs_encoding = sys.getfilesystemencoding() + while 1: + line = stdout.readline() + try: + line = line.decode(encoding) + except UnicodeDecodeError: + line = line.decode(fs_encoding) + if not line: + break + line = line.rstrip() + all_output.append(line) + if filter_stdout: + level = filter_stdout(line) + if isinstance(level, tuple): + level, line = level + logger.log(level, line) + if not logger.stdout_level_matches(level): + logger.show_progress() + else: + logger.info(line) + else: + proc.communicate(stdin) + proc.wait() + if proc.returncode: + if raise_on_returncode: + if all_output: + logger.notify('Complete output from command %s:' % cmd_desc) + logger.notify('\n'.join(all_output) + '\n----------------------------------------') + raise OSError( + "Command %s failed with error code %s" + % (cmd_desc, proc.returncode)) + else: + logger.warn( + "Command %s had error code %s" + % (cmd_desc, proc.returncode)) + +def filter_install_output(line): + if line.strip().startswith('running'): + return Logger.INFO + return Logger.DEBUG + +def find_wheels(projects, search_dirs): + """Find wheels from which we can import PROJECTS. + + Scan through SEARCH_DIRS for a wheel for each PROJECT in turn. Return + a list of the first wheel found for each PROJECT + """ + + wheels = [] + + # Look through SEARCH_DIRS for the first suitable wheel. Don't bother + # about version checking here, as this is simply to get something we can + # then use to install the correct version. + for project in projects: + for dirname in search_dirs: + # This relies on only having "universal" wheels available. + # The pattern could be tightened to require -py2.py3-none-any.whl. + files = glob.glob(os.path.join(dirname, project + '-*.whl')) + if files: + wheels.append(os.path.abspath(files[0])) + break + else: + # We're out of luck, so quit with a suitable error + logger.fatal('Cannot find a wheel for %s' % (project,)) + + return wheels + +def install_wheel(project_names, py_executable, search_dirs=None, + download=False): + if search_dirs is None: + search_dirs = file_search_dirs() + + wheels = find_wheels(['setuptools', 'pip'], search_dirs) + pythonpath = os.pathsep.join(wheels) + + # PIP_FIND_LINKS uses space as the path separator and thus cannot have paths + # with spaces in them. Convert any of those to local file:// URL form. + try: + from urlparse import urljoin + from urllib import pathname2url + except ImportError: + from urllib.parse import urljoin + from urllib.request import pathname2url + def space_path2url(p): + if ' ' not in p: + return p + return urljoin('file:', pathname2url(os.path.abspath(p))) + findlinks = ' '.join(space_path2url(d) for d in search_dirs) + + SCRIPT = textwrap.dedent(""" + import sys + import pkgutil + import tempfile + import os + + import pip + + cert_data = pkgutil.get_data("pip._vendor.requests", "cacert.pem") + if cert_data is not None: + cert_file = tempfile.NamedTemporaryFile(delete=False) + cert_file.write(cert_data) + cert_file.close() + else: + cert_file = None + + try: + args = ["install", "--ignore-installed"] + if cert_file is not None: + args += ["--cert", cert_file.name] + args += sys.argv[1:] + + sys.exit(pip.main(args)) + finally: + if cert_file is not None: + os.remove(cert_file.name) + """).encode("utf8") + + cmd = [py_executable, '-'] + project_names + logger.start_progress('Installing %s...' % (', '.join(project_names))) + logger.indent += 2 + + env = { + "PYTHONPATH": pythonpath, + "JYTHONPATH": pythonpath, # for Jython < 3.x + "PIP_FIND_LINKS": findlinks, + "PIP_USE_WHEEL": "1", + "PIP_ONLY_BINARY": ":all:", + "PIP_PRE": "1", + "PIP_USER": "0", + } + + if not download: + env["PIP_NO_INDEX"] = "1" + + try: + call_subprocess(cmd, show_stdout=False, extra_env=env, stdin=SCRIPT) + finally: + logger.indent -= 2 + logger.end_progress() + + +def create_environment(home_dir, site_packages=False, clear=False, + unzip_setuptools=False, + prompt=None, search_dirs=None, download=False, + no_setuptools=False, no_pip=False, no_wheel=False, + symlink=True): + """ + Creates a new environment in ``home_dir``. + + If ``site_packages`` is true, then the global ``site-packages/`` + directory will be on the path. + + If ``clear`` is true (default False) then the environment will + first be cleared. + """ + home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) + + py_executable = os.path.abspath(install_python( + home_dir, lib_dir, inc_dir, bin_dir, + site_packages=site_packages, clear=clear, symlink=symlink)) + + install_distutils(home_dir) + + to_install = [] + + if not no_setuptools: + to_install.append('setuptools') + + if not no_pip: + to_install.append('pip') + + if not no_wheel: + to_install.append('wheel') + + if to_install: + install_wheel( + to_install, + py_executable, + search_dirs, + download=download, + ) + + install_activate(home_dir, bin_dir, prompt) + + install_python_config(home_dir, bin_dir, prompt) + +def is_executable_file(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def path_locations(home_dir): + """Return the path locations for the environment (where libraries are, + where scripts go, etc)""" + home_dir = os.path.abspath(home_dir) + # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its + # prefix arg is broken: http://bugs.python.org/issue3386 + if is_win: + # Windows has lots of problems with executables with spaces in + # the name; this function will remove them (using the ~1 + # format): + mkdir(home_dir) + if ' ' in home_dir: + import ctypes + GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW + size = max(len(home_dir)+1, 256) + buf = ctypes.create_unicode_buffer(size) + try: + u = unicode + except NameError: + u = str + ret = GetShortPathName(u(home_dir), buf, size) + if not ret: + print('Error: the path "%s" has a space in it' % home_dir) + print('We could not determine the short pathname for it.') + print('Exiting.') + sys.exit(3) + home_dir = str(buf.value) + lib_dir = join(home_dir, 'Lib') + inc_dir = join(home_dir, 'Include') + bin_dir = join(home_dir, 'Scripts') + if is_jython: + lib_dir = join(home_dir, 'Lib') + inc_dir = join(home_dir, 'Include') + bin_dir = join(home_dir, 'bin') + elif is_pypy: + lib_dir = home_dir + inc_dir = join(home_dir, 'include') + bin_dir = join(home_dir, 'bin') + elif not is_win: + lib_dir = join(home_dir, 'lib', py_version) + inc_dir = join(home_dir, 'include', py_version + abiflags) + bin_dir = join(home_dir, 'bin') + return home_dir, lib_dir, inc_dir, bin_dir + + +def change_prefix(filename, dst_prefix): + prefixes = [sys.prefix] + + if is_darwin: + prefixes.extend(( + os.path.join("/Library/Python", sys.version[:3], "site-packages"), + os.path.join(sys.prefix, "Extras", "lib", "python"), + os.path.join("~", "Library", "Python", sys.version[:3], "site-packages"), + # Python 2.6 no-frameworks + os.path.join("~", ".local", "lib","python", sys.version[:3], "site-packages"), + # System Python 2.7 on OSX Mountain Lion + os.path.join("~", "Library", "Python", sys.version[:3], "lib", "python", "site-packages"))) + + if hasattr(sys, 'real_prefix'): + prefixes.append(sys.real_prefix) + if hasattr(sys, 'base_prefix'): + prefixes.append(sys.base_prefix) + prefixes = list(map(os.path.expanduser, prefixes)) + prefixes = list(map(os.path.abspath, prefixes)) + # Check longer prefixes first so we don't split in the middle of a filename + prefixes = sorted(prefixes, key=len, reverse=True) + filename = os.path.abspath(filename) + # On Windows, make sure drive letter is uppercase + if is_win and filename[0] in 'abcdefghijklmnopqrstuvwxyz': + filename = filename[0].upper() + filename[1:] + for i, prefix in enumerate(prefixes): + if is_win and prefix[0] in 'abcdefghijklmnopqrstuvwxyz': + prefixes[i] = prefix[0].upper() + prefix[1:] + for src_prefix in prefixes: + if filename.startswith(src_prefix): + _, relpath = filename.split(src_prefix, 1) + if src_prefix != os.sep: # sys.prefix == "/" + assert relpath[0] == os.sep + relpath = relpath[1:] + return join(dst_prefix, relpath) + assert False, "Filename %s does not start with any of these prefixes: %s" % \ + (filename, prefixes) + +def copy_required_modules(dst_prefix, symlink): + import imp + + for modname in REQUIRED_MODULES: + if modname in sys.builtin_module_names: + logger.info("Ignoring built-in bootstrap module: %s" % modname) + continue + try: + f, filename, _ = imp.find_module(modname) + except ImportError: + logger.info("Cannot import bootstrap module: %s" % modname) + else: + if f is not None: + f.close() + # special-case custom readline.so on OS X, but not for pypy: + if modname == 'readline' and sys.platform == 'darwin' and not ( + is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))): + dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so') + elif modname == 'readline' and sys.platform == 'win32': + # special-case for Windows, where readline is not a + # standard module, though it may have been installed in + # site-packages by a third-party package + pass + else: + dst_filename = change_prefix(filename, dst_prefix) + copyfile(filename, dst_filename, symlink) + if filename.endswith('.pyc'): + pyfile = filename[:-1] + if os.path.exists(pyfile): + copyfile(pyfile, dst_filename[:-1], symlink) + + +def subst_path(prefix_path, prefix, home_dir): + prefix_path = os.path.normpath(prefix_path) + prefix = os.path.normpath(prefix) + home_dir = os.path.normpath(home_dir) + if not prefix_path.startswith(prefix): + logger.warn('Path not in prefix %r %r', prefix_path, prefix) + return + return prefix_path.replace(prefix, home_dir, 1) + + +def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, symlink=True): + """Install just the base environment, no distutils patches etc""" + if sys.executable.startswith(bin_dir): + print('Please use the *system* python to run this script') + return + + if clear: + rmtree(lib_dir) + ## FIXME: why not delete it? + ## Maybe it should delete everything with #!/path/to/venv/python in it + logger.notify('Not deleting %s', bin_dir) + + if hasattr(sys, 'real_prefix'): + logger.notify('Using real prefix %r' % sys.real_prefix) + prefix = sys.real_prefix + elif hasattr(sys, 'base_prefix'): + logger.notify('Using base prefix %r' % sys.base_prefix) + prefix = sys.base_prefix + else: + prefix = sys.prefix + mkdir(lib_dir) + fix_lib64(lib_dir, symlink) + stdlib_dirs = [os.path.dirname(os.__file__)] + if is_win: + stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), 'DLLs')) + elif is_darwin: + stdlib_dirs.append(join(stdlib_dirs[0], 'site-packages')) + if hasattr(os, 'symlink'): + logger.info('Symlinking Python bootstrap modules') + else: + logger.info('Copying Python bootstrap modules') + logger.indent += 2 + try: + # copy required files... + for stdlib_dir in stdlib_dirs: + if not os.path.isdir(stdlib_dir): + continue + for fn in os.listdir(stdlib_dir): + bn = os.path.splitext(fn)[0] + if fn != 'site-packages' and bn in REQUIRED_FILES: + copyfile(join(stdlib_dir, fn), join(lib_dir, fn), symlink) + # ...and modules + copy_required_modules(home_dir, symlink) + finally: + logger.indent -= 2 + mkdir(join(lib_dir, 'site-packages')) + import site + site_filename = site.__file__ + if site_filename.endswith('.pyc') or site_filename.endswith('.pyo'): + site_filename = site_filename[:-1] + elif site_filename.endswith('$py.class'): + site_filename = site_filename.replace('$py.class', '.py') + site_filename_dst = change_prefix(site_filename, home_dir) + site_dir = os.path.dirname(site_filename_dst) + # MOZ: Copies a site.py if it exists instead of using the one hex encoded in + # this file. Necessary for some site.py fixes for MinGW64 version of python + site_py_src_path = os.path.join(os.path.dirname(__file__), 'site.py') + if os.path.isfile(site_py_src_path): + shutil.copy(site_py_src_path, site_filename_dst) + else: + writefile(site_filename_dst, SITE_PY) + writefile(join(site_dir, 'orig-prefix.txt'), prefix) + site_packages_filename = join(site_dir, 'no-global-site-packages.txt') + if not site_packages: + writefile(site_packages_filename, '') + + if is_pypy or is_win: + stdinc_dir = join(prefix, 'include') + else: + stdinc_dir = join(prefix, 'include', py_version + abiflags) + if os.path.exists(stdinc_dir): + copyfile(stdinc_dir, inc_dir, symlink) + else: + logger.debug('No include dir %s' % stdinc_dir) + + platinc_dir = distutils.sysconfig.get_python_inc(plat_specific=1) + if platinc_dir != stdinc_dir: + platinc_dest = distutils.sysconfig.get_python_inc( + plat_specific=1, prefix=home_dir) + if platinc_dir == platinc_dest: + # Do platinc_dest manually due to a CPython bug; + # not http://bugs.python.org/issue3386 but a close cousin + platinc_dest = subst_path(platinc_dir, prefix, home_dir) + if platinc_dest: + # PyPy's stdinc_dir and prefix are relative to the original binary + # (traversing virtualenvs), whereas the platinc_dir is relative to + # the inner virtualenv and ignores the prefix argument. + # This seems more evolved than designed. + copyfile(platinc_dir, platinc_dest, symlink) + + # pypy never uses exec_prefix, just ignore it + if sys.exec_prefix != prefix and not is_pypy: + if is_win: + exec_dir = join(sys.exec_prefix, 'lib') + elif is_jython: + exec_dir = join(sys.exec_prefix, 'Lib') + else: + exec_dir = join(sys.exec_prefix, 'lib', py_version) + for fn in os.listdir(exec_dir): + copyfile(join(exec_dir, fn), join(lib_dir, fn), symlink) + + if is_jython: + # Jython has either jython-dev.jar and javalib/ dir, or just + # jython.jar + for name in 'jython-dev.jar', 'javalib', 'jython.jar': + src = join(prefix, name) + if os.path.exists(src): + copyfile(src, join(home_dir, name), symlink) + # XXX: registry should always exist after Jython 2.5rc1 + src = join(prefix, 'registry') + if os.path.exists(src): + copyfile(src, join(home_dir, 'registry'), symlink=False) + copyfile(join(prefix, 'cachedir'), join(home_dir, 'cachedir'), + symlink=False) + + mkdir(bin_dir) + py_executable = join(bin_dir, os.path.basename(sys.executable)) + if 'Python.framework' in prefix: + # OS X framework builds cause validation to break + # https://github.com/pypa/virtualenv/issues/322 + if os.environ.get('__PYVENV_LAUNCHER__'): + del os.environ["__PYVENV_LAUNCHER__"] + if re.search(r'/Python(?:-32|-64)*$', py_executable): + # The name of the python executable is not quite what + # we want, rename it. + py_executable = os.path.join( + os.path.dirname(py_executable), 'python') + + logger.notify('New %s executable in %s', expected_exe, py_executable) + pcbuild_dir = os.path.dirname(sys.executable) + pyd_pth = os.path.join(lib_dir, 'site-packages', 'virtualenv_builddir_pyd.pth') + if is_win and os.path.exists(os.path.join(pcbuild_dir, 'build.bat')): + logger.notify('Detected python running from build directory %s', pcbuild_dir) + logger.notify('Writing .pth file linking to build directory for *.pyd files') + writefile(pyd_pth, pcbuild_dir) + else: + pcbuild_dir = None + if os.path.exists(pyd_pth): + logger.info('Deleting %s (not Windows env or not build directory python)' % pyd_pth) + os.unlink(pyd_pth) + + if sys.executable != py_executable: + ## FIXME: could I just hard link? + executable = sys.executable + shutil.copyfile(executable, py_executable) + make_exe(py_executable) + if is_win or is_cygwin: + pythonw = os.path.join(os.path.dirname(sys.executable), 'pythonw.exe') + if os.path.exists(pythonw): + logger.info('Also created pythonw.exe') + shutil.copyfile(pythonw, os.path.join(os.path.dirname(py_executable), 'pythonw.exe')) + python_d = os.path.join(os.path.dirname(sys.executable), 'python_d.exe') + python_d_dest = os.path.join(os.path.dirname(py_executable), 'python_d.exe') + if os.path.exists(python_d): + logger.info('Also created python_d.exe') + shutil.copyfile(python_d, python_d_dest) + elif os.path.exists(python_d_dest): + logger.info('Removed python_d.exe as it is no longer at the source') + os.unlink(python_d_dest) + # we need to copy the DLL to enforce that windows will load the correct one. + # may not exist if we are cygwin. + py_executable_dll = 'python%s%s.dll' % ( + sys.version_info[0], sys.version_info[1]) + py_executable_dll_d = 'python%s%s_d.dll' % ( + sys.version_info[0], sys.version_info[1]) + pythondll = os.path.join(os.path.dirname(sys.executable), py_executable_dll) + pythondll_d = os.path.join(os.path.dirname(sys.executable), py_executable_dll_d) + pythondll_d_dest = os.path.join(os.path.dirname(py_executable), py_executable_dll_d) + if os.path.exists(pythondll): + logger.info('Also created %s' % py_executable_dll) + shutil.copyfile(pythondll, os.path.join(os.path.dirname(py_executable), py_executable_dll)) + if os.path.exists(pythondll_d): + logger.info('Also created %s' % py_executable_dll_d) + shutil.copyfile(pythondll_d, pythondll_d_dest) + elif os.path.exists(pythondll_d_dest): + logger.info('Removed %s as the source does not exist' % pythondll_d_dest) + os.unlink(pythondll_d_dest) + if is_pypy: + # make a symlink python --> pypy-c + python_executable = os.path.join(os.path.dirname(py_executable), 'python') + if sys.platform in ('win32', 'cygwin'): + python_executable += '.exe' + logger.info('Also created executable %s' % python_executable) + copyfile(py_executable, python_executable, symlink) + + if is_win: + for name in ['libexpat.dll', 'libpypy.dll', 'libpypy-c.dll', + 'libeay32.dll', 'ssleay32.dll', 'sqlite3.dll', + 'tcl85.dll', 'tk85.dll']: + src = join(prefix, name) + if os.path.exists(src): + copyfile(src, join(bin_dir, name), symlink) + + for d in sys.path: + if d.endswith('lib_pypy'): + break + else: + logger.fatal('Could not find lib_pypy in sys.path') + raise SystemExit(3) + logger.info('Copying lib_pypy') + copyfile(d, os.path.join(home_dir, 'lib_pypy'), symlink) + + if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe: + secondary_exe = os.path.join(os.path.dirname(py_executable), + expected_exe) + py_executable_ext = os.path.splitext(py_executable)[1] + if py_executable_ext.lower() == '.exe': + # python2.4 gives an extension of '.4' :P + secondary_exe += py_executable_ext + if os.path.exists(secondary_exe): + logger.warn('Not overwriting existing %s script %s (you must use %s)' + % (expected_exe, secondary_exe, py_executable)) + else: + logger.notify('Also creating executable in %s' % secondary_exe) + shutil.copyfile(sys.executable, secondary_exe) + make_exe(secondary_exe) + + if '.framework' in prefix: + if 'Python.framework' in prefix: + logger.debug('MacOSX Python framework detected') + # Make sure we use the embedded interpreter inside + # the framework, even if sys.executable points to + # the stub executable in ${sys.prefix}/bin + # See http://groups.google.com/group/python-virtualenv/ + # browse_thread/thread/17cab2f85da75951 + original_python = os.path.join( + prefix, 'Resources/Python.app/Contents/MacOS/Python') + if 'EPD' in prefix: + logger.debug('EPD framework detected') + original_python = os.path.join(prefix, 'bin/python') + shutil.copy(original_python, py_executable) + + # Copy the framework's dylib into the virtual + # environment + virtual_lib = os.path.join(home_dir, '.Python') + + if os.path.exists(virtual_lib): + os.unlink(virtual_lib) + copyfile( + os.path.join(prefix, 'Python'), + virtual_lib, + symlink) + + # And then change the install_name of the copied python executable + try: + mach_o_change(py_executable, + os.path.join(prefix, 'Python'), + '@executable_path/../.Python') + except: + e = sys.exc_info()[1] + logger.warn("Could not call mach_o_change: %s. " + "Trying to call install_name_tool instead." % e) + try: + call_subprocess( + ["install_name_tool", "-change", + os.path.join(prefix, 'Python'), + '@executable_path/../.Python', + py_executable]) + except: + logger.fatal("Could not call install_name_tool -- you must " + "have Apple's development tools installed") + raise + + if not is_win: + # Ensure that 'python', 'pythonX' and 'pythonX.Y' all exist + py_exe_version_major = 'python%s' % sys.version_info[0] + py_exe_version_major_minor = 'python%s.%s' % ( + sys.version_info[0], sys.version_info[1]) + py_exe_no_version = 'python' + required_symlinks = [ py_exe_no_version, py_exe_version_major, + py_exe_version_major_minor ] + + py_executable_base = os.path.basename(py_executable) + + if py_executable_base in required_symlinks: + # Don't try to symlink to yourself. + required_symlinks.remove(py_executable_base) + + for pth in required_symlinks: + full_pth = join(bin_dir, pth) + if os.path.exists(full_pth): + os.unlink(full_pth) + if symlink: + os.symlink(py_executable_base, full_pth) + else: + copyfile(py_executable, full_pth, symlink) + + if is_win and ' ' in py_executable: + # There's a bug with subprocess on Windows when using a first + # argument that has a space in it. Instead we have to quote + # the value: + py_executable = '"%s"' % py_executable + # NOTE: keep this check as one line, cmd.exe doesn't cope with line breaks + cmd = [py_executable, '-c', 'import sys;out=sys.stdout;' + 'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))'] + logger.info('Testing executable with %s %s "%s"' % tuple(cmd)) + try: + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE) + proc_stdout, proc_stderr = proc.communicate() + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.EACCES: + logger.fatal('ERROR: The executable %s could not be run: %s' % (py_executable, e)) + sys.exit(100) + else: + raise e + + proc_stdout = proc_stdout.strip().decode("utf-8") + proc_stdout = os.path.normcase(os.path.abspath(proc_stdout)) + norm_home_dir = os.path.normcase(os.path.abspath(home_dir)) + if hasattr(norm_home_dir, 'decode'): + norm_home_dir = norm_home_dir.decode(sys.getfilesystemencoding()) + if proc_stdout != norm_home_dir: + logger.fatal( + 'ERROR: The executable %s is not functioning' % py_executable) + logger.fatal( + 'ERROR: It thinks sys.prefix is %r (should be %r)' + % (proc_stdout, norm_home_dir)) + logger.fatal( + 'ERROR: virtualenv is not compatible with this system or executable') + if is_win: + logger.fatal( + 'Note: some Windows users have reported this error when they ' + 'installed Python for "Only this user" or have multiple ' + 'versions of Python installed. Copying the appropriate ' + 'PythonXX.dll to the virtualenv Scripts/ directory may fix ' + 'this problem.') + sys.exit(100) + else: + logger.info('Got sys.prefix result: %r' % proc_stdout) + + pydistutils = os.path.expanduser('~/.pydistutils.cfg') + if os.path.exists(pydistutils): + logger.notify('Please make sure you remove any previous custom paths from ' + 'your %s file.' % pydistutils) + ## FIXME: really this should be calculated earlier + + fix_local_scheme(home_dir, symlink) + + if site_packages: + if os.path.exists(site_packages_filename): + logger.info('Deleting %s' % site_packages_filename) + os.unlink(site_packages_filename) + + return py_executable + + +def install_activate(home_dir, bin_dir, prompt=None): + if is_win or is_jython and os._name == 'nt': + files = { + 'activate.bat': ACTIVATE_BAT, + 'deactivate.bat': DEACTIVATE_BAT, + 'activate.ps1': ACTIVATE_PS, + } + + # MSYS needs paths of the form /c/path/to/file + drive, tail = os.path.splitdrive(home_dir.replace(os.sep, '/')) + home_dir_msys = (drive and "/%s%s" or "%s%s") % (drive[:1], tail) + + # Run-time conditional enables (basic) Cygwin compatibility + home_dir_sh = ("""$(if [ "$OSTYPE" "==" "cygwin" ]; then cygpath -u '%s'; else echo '%s'; fi;)""" % + (home_dir, home_dir_msys)) + files['activate'] = ACTIVATE_SH.replace('__VIRTUAL_ENV__', home_dir_sh) + + else: + files = {'activate': ACTIVATE_SH} + + # suppling activate.fish in addition to, not instead of, the + # bash script support. + files['activate.fish'] = ACTIVATE_FISH + + # same for csh/tcsh support... + files['activate.csh'] = ACTIVATE_CSH + + files['activate_this.py'] = ACTIVATE_THIS + + install_files(home_dir, bin_dir, prompt, files) + +def install_files(home_dir, bin_dir, prompt, files): + if hasattr(home_dir, 'decode'): + home_dir = home_dir.decode(sys.getfilesystemencoding()) + vname = os.path.basename(home_dir) + for name, content in files.items(): + content = content.replace('__VIRTUAL_PROMPT__', prompt or '') + content = content.replace('__VIRTUAL_WINPROMPT__', prompt or '(%s)' % vname) + content = content.replace('__VIRTUAL_ENV__', home_dir) + content = content.replace('__VIRTUAL_NAME__', vname) + content = content.replace('__BIN_NAME__', os.path.basename(bin_dir)) + writefile(os.path.join(bin_dir, name), content) + +def install_python_config(home_dir, bin_dir, prompt=None): + if sys.platform == 'win32' or is_jython and os._name == 'nt': + files = {} + else: + files = {'python-config': PYTHON_CONFIG} + install_files(home_dir, bin_dir, prompt, files) + for name, content in files.items(): + make_exe(os.path.join(bin_dir, name)) + +def install_distutils(home_dir): + distutils_path = change_prefix(distutils.__path__[0], home_dir) + mkdir(distutils_path) + ## FIXME: maybe this prefix setting should only be put in place if + ## there's a local distutils.cfg with a prefix setting? + home_dir = os.path.abspath(home_dir) + ## FIXME: this is breaking things, removing for now: + #distutils_cfg = DISTUTILS_CFG + "\n[install]\nprefix=%s\n" % home_dir + writefile(os.path.join(distutils_path, '__init__.py'), DISTUTILS_INIT) + writefile(os.path.join(distutils_path, 'distutils.cfg'), DISTUTILS_CFG, overwrite=False) + +def fix_local_scheme(home_dir, symlink=True): + """ + Platforms that use the "posix_local" install scheme (like Ubuntu with + Python 2.7) need to be given an additional "local" location, sigh. + """ + try: + import sysconfig + except ImportError: + pass + else: + if sysconfig._get_default_scheme() == 'posix_local': + local_path = os.path.join(home_dir, 'local') + if not os.path.exists(local_path): + os.mkdir(local_path) + for subdir_name in os.listdir(home_dir): + if subdir_name == 'local': + continue + copyfile(os.path.abspath(os.path.join(home_dir, subdir_name)), \ + os.path.join(local_path, subdir_name), symlink) + +def fix_lib64(lib_dir, symlink=True): + """ + Some platforms (particularly Gentoo on x64) put things in lib64/pythonX.Y + instead of lib/pythonX.Y. If this is such a platform we'll just create a + symlink so lib64 points to lib + """ + # PyPy's library path scheme is not affected by this. + # Return early or we will die on the following assert. + if is_pypy: + logger.debug('PyPy detected, skipping lib64 symlinking') + return + # Check we have a lib64 library path + if not [p for p in distutils.sysconfig.get_config_vars().values() + if isinstance(p, basestring) and 'lib64' in p]: + return + + logger.debug('This system uses lib64; symlinking lib64 to lib') + + assert os.path.basename(lib_dir) == 'python%s' % sys.version[:3], ( + "Unexpected python lib dir: %r" % lib_dir) + lib_parent = os.path.dirname(lib_dir) + top_level = os.path.dirname(lib_parent) + lib_dir = os.path.join(top_level, 'lib') + lib64_link = os.path.join(top_level, 'lib64') + assert os.path.basename(lib_parent) == 'lib', ( + "Unexpected parent dir: %r" % lib_parent) + if os.path.lexists(lib64_link): + return + if symlink: + os.symlink('lib', lib64_link) + else: + copyfile('lib', lib64_link) + +def resolve_interpreter(exe): + """ + If the executable given isn't an absolute path, search $PATH for the interpreter + """ + # If the "executable" is a version number, get the installed executable for + # that version + python_versions = get_installed_pythons() + if exe in python_versions: + exe = python_versions[exe] + + if os.path.abspath(exe) != exe: + paths = os.environ.get('PATH', '').split(os.pathsep) + for path in paths: + if os.path.exists(join(path, exe)): + exe = join(path, exe) + break + if not os.path.exists(exe): + logger.fatal('The executable %s (from --python=%s) does not exist' % (exe, exe)) + raise SystemExit(3) + if not is_executable(exe): + logger.fatal('The executable %s (from --python=%s) is not executable' % (exe, exe)) + raise SystemExit(3) + return exe + +def is_executable(exe): + """Checks a file is executable""" + return os.access(exe, os.X_OK) + +############################################################ +## Relocating the environment: + +def make_environment_relocatable(home_dir): + """ + Makes the already-existing environment use relative paths, and takes out + the #!-based environment selection in scripts. + """ + home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) + activate_this = os.path.join(bin_dir, 'activate_this.py') + if not os.path.exists(activate_this): + logger.fatal( + 'The environment doesn\'t have a file %s -- please re-run virtualenv ' + 'on this environment to update it' % activate_this) + fixup_scripts(home_dir, bin_dir) + fixup_pth_and_egg_link(home_dir) + ## FIXME: need to fix up distutils.cfg + +OK_ABS_SCRIPTS = ['python', 'python%s' % sys.version[:3], + 'activate', 'activate.bat', 'activate_this.py', + 'activate.fish', 'activate.csh'] + +def fixup_scripts(home_dir, bin_dir): + if is_win: + new_shebang_args = ( + '%s /c' % os.path.normcase(os.environ.get('COMSPEC', 'cmd.exe')), + '', '.exe') + else: + new_shebang_args = ('/usr/bin/env', sys.version[:3], '') + + # This is what we expect at the top of scripts: + shebang = '#!%s' % os.path.normcase(os.path.join( + os.path.abspath(bin_dir), 'python%s' % new_shebang_args[2])) + # This is what we'll put: + new_shebang = '#!%s python%s%s' % new_shebang_args + + for filename in os.listdir(bin_dir): + filename = os.path.join(bin_dir, filename) + if not os.path.isfile(filename): + # ignore subdirs, e.g. .svn ones. + continue + lines = None + with open(filename, 'rb') as f: + try: + lines = f.read().decode('utf-8').splitlines() + except UnicodeDecodeError: + # This is probably a binary program instead + # of a script, so just ignore it. + continue + if not lines: + logger.warn('Script %s is an empty file' % filename) + continue + + old_shebang = lines[0].strip() + old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:]) + + if not old_shebang.startswith(shebang): + if os.path.basename(filename) in OK_ABS_SCRIPTS: + logger.debug('Cannot make script %s relative' % filename) + elif lines[0].strip() == new_shebang: + logger.info('Script %s has already been made relative' % filename) + else: + logger.warn('Script %s cannot be made relative (it\'s not a normal script that starts with %s)' + % (filename, shebang)) + continue + logger.notify('Making script %s relative' % filename) + script = relative_script([new_shebang] + lines[1:]) + with open(filename, 'wb') as f: + f.write('\n'.join(script).encode('utf-8')) + + +def relative_script(lines): + "Return a script that'll work in a relocatable environment." + activate = "import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del os, activate_this" + # Find the last future statement in the script. If we insert the activation + # line before a future statement, Python will raise a SyntaxError. + activate_at = None + for idx, line in reversed(list(enumerate(lines))): + if line.split()[:3] == ['from', '__future__', 'import']: + activate_at = idx + 1 + break + if activate_at is None: + # Activate after the shebang. + activate_at = 1 + return lines[:activate_at] + ['', activate, ''] + lines[activate_at:] + +def fixup_pth_and_egg_link(home_dir, sys_path=None): + """Makes .pth and .egg-link files use relative paths""" + home_dir = os.path.normcase(os.path.abspath(home_dir)) + if sys_path is None: + sys_path = sys.path + for path in sys_path: + if not path: + path = '.' + if not os.path.isdir(path): + continue + path = os.path.normcase(os.path.abspath(path)) + if not path.startswith(home_dir): + logger.debug('Skipping system (non-environment) directory %s' % path) + continue + for filename in os.listdir(path): + filename = os.path.join(path, filename) + if filename.endswith('.pth'): + if not os.access(filename, os.W_OK): + logger.warn('Cannot write .pth file %s, skipping' % filename) + else: + fixup_pth_file(filename) + if filename.endswith('.egg-link'): + if not os.access(filename, os.W_OK): + logger.warn('Cannot write .egg-link file %s, skipping' % filename) + else: + fixup_egg_link(filename) + +def fixup_pth_file(filename): + lines = [] + prev_lines = [] + with open(filename) as f: + prev_lines = f.readlines() + for line in prev_lines: + line = line.strip() + if (not line or line.startswith('#') or line.startswith('import ') + or os.path.abspath(line) != line): + lines.append(line) + else: + new_value = make_relative_path(filename, line) + if line != new_value: + logger.debug('Rewriting path %s as %s (in %s)' % (line, new_value, filename)) + lines.append(new_value) + if lines == prev_lines: + logger.info('No changes to .pth file %s' % filename) + return + logger.notify('Making paths in .pth file %s relative' % filename) + with open(filename, 'w') as f: + f.write('\n'.join(lines) + '\n') + +def fixup_egg_link(filename): + with open(filename) as f: + link = f.readline().strip() + if os.path.abspath(link) != link: + logger.debug('Link in %s already relative' % filename) + return + new_link = make_relative_path(filename, link) + logger.notify('Rewriting link %s in %s as %s' % (link, filename, new_link)) + with open(filename, 'w') as f: + f.write(new_link) + +def make_relative_path(source, dest, dest_is_directory=True): + """ + Make a filename relative, where the filename is dest, and it is + being referred to from the filename source. + + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/usr/share/another-place/src/Directory') + '../another-place/src/Directory' + >>> make_relative_path('/usr/share/something/a-file.pth', + ... '/home/user/src/Directory') + '../../../home/user/src/Directory' + >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') + './' + """ + source = os.path.dirname(source) + if not dest_is_directory: + dest_filename = os.path.basename(dest) + dest = os.path.dirname(dest) + dest = os.path.normpath(os.path.abspath(dest)) + source = os.path.normpath(os.path.abspath(source)) + dest_parts = dest.strip(os.path.sep).split(os.path.sep) + source_parts = source.strip(os.path.sep).split(os.path.sep) + while dest_parts and source_parts and dest_parts[0] == source_parts[0]: + dest_parts.pop(0) + source_parts.pop(0) + full_parts = ['..']*len(source_parts) + dest_parts + if not dest_is_directory: + full_parts.append(dest_filename) + if not full_parts: + # Special case for the current directory (otherwise it'd be '') + return './' + return os.path.sep.join(full_parts) + + + +############################################################ +## Bootstrap script creation: + +def create_bootstrap_script(extra_text, python_version=''): + """ + Creates a bootstrap script, which is like this script but with + extend_parser, adjust_options, and after_install hooks. + + This returns a string that (written to disk of course) can be used + as a bootstrap script with your own customizations. The script + will be the standard virtualenv.py script, with your extra text + added (your extra text should be Python code). + + If you include these functions, they will be called: + + ``extend_parser(optparse_parser)``: + You can add or remove options from the parser here. + + ``adjust_options(options, args)``: + You can change options here, or change the args (if you accept + different kinds of arguments, be sure you modify ``args`` so it is + only ``[DEST_DIR]``). + + ``after_install(options, home_dir)``: + + After everything is installed, this function is called. This + is probably the function you are most likely to use. An + example would be:: + + def after_install(options, home_dir): + subprocess.call([join(home_dir, 'bin', 'easy_install'), + 'MyPackage']) + subprocess.call([join(home_dir, 'bin', 'my-package-script'), + 'setup', home_dir]) + + This example immediately installs a package, and runs a setup + script from that package. + + If you provide something like ``python_version='2.5'`` then the + script will start with ``#!/usr/bin/env python2.5`` instead of + ``#!/usr/bin/env python``. You can use this when the script must + be run with a particular Python version. + """ + filename = __file__ + if filename.endswith('.pyc'): + filename = filename[:-1] + with codecs.open(filename, 'r', encoding='utf-8') as f: + content = f.read() + py_exe = 'python%s' % python_version + content = (('#!/usr/bin/env %s\n' % py_exe) + + '## WARNING: This file is generated\n' + + content) + return content.replace('##EXT' 'END##', extra_text) + +##EXTEND## + +def convert(s): + b = base64.b64decode(s.encode('ascii')) + return zlib.decompress(b).decode('utf-8') + +##file site.py +SITE_PY = convert(""" +eJzFPf1z2zaWv/OvwMqToZTKdOJ0e3tO3RsncVrfuYm3yc7m1vXoKAmyWFMkS5C2tTd3f/u9DwAE ++CHb2+6cphNLJPDw8PC+8PAeOhqNTopCZkuxyZd1KoWScblYiyKu1kqs8lJU66Rc7hdxWW3h6eIm +vpZKVLlQWxVhqygInv/GT/BcfF4nyqAA3+K6yjdxlSziNN2KZFPkZSWXYlmXSXYtkiypkjhN/g4t +8iwSz387BsFZJmDmaSJLcStLBXCVyFfiYlut80yM6wLn/DL6Y/xqMhVqUSZFBQ1KjTNQZB1XQSbl +EtCElrUCUiaV3FeFXCSrZGEb3uV1uhRFGi+k+K//4qlR0zAMVL6Rd2tZSpEBMgBTAqwC8YCvSSkW ++VJGQryRixgH4OcNsQKGNsU1U0jGLBdpnl3DnDK5kErF5VaM53VFgAhlscwBpwQwqJI0De7y8kZN +YElpPe7gkYiZPfzJMHvAPHH8LucAjh+z4C9Zcj9l2MA9CK5aM9uUcpXcixjBwk95Lxcz/WycrMQy +Wa2ABlk1wSYBI6BEmswPClqOb/UKfXdAWFmujGEMiShzY35JPaLgrBJxqoBt6wJppAjzd3KexBlQ +I7uF4QAikDToG2eZqMqOQ7MTOQAocR0rkJKNEuNNnGTArD/GC0L7r0m2zO/UhCgAq6XEL7Wq3PmP +ewgArR0CTANcLLOadZYmNzLdTgCBz4B9KVWdVigQy6SUiyovE6kIAKC2FfIekJ6KuJSahMyZRm6n +RH+iSZLhwqKAocDjSyTJKrmuS5IwsUqAc4Er3n/8Sbw7fXN28kHzmAHGMnu9AZwBCi20gxMMIA5q +VR6kOQh0FJzjHxEvlyhk1zg+4NU0OHhwpYMxzL2I2n2cBQey68XVw8AcK1AmNFZA/f4bukzVGujz +Pw+sdxCcDFGFJs7f7tY5yGQWb6RYx8xfyBnBtxrOd1FRrV8DNyiEUwGpFC4OIpggPCCJS7NxnklR +AIulSSYnAVBoTm39VQRW+JBn+7TWLU4ACGWQwUvn2YRGzCRMtAvrNeoL03hLM9NNArvOm7wkxQH8 +ny1IF6VxdkM4KmIo/jaX10mWIULIC0G4F9LA6iYBTlxG4pxakV4wjUTI2otbokjUwEvIdMCT8j7e +FKmcsviibt2tRmgwWQmz1ilzHLSsSL3SqjVT7eW9w+hLi+sIzWpdSgBezz2hW+X5VMxBZxM2Rbxh +8arucuKcoEeeqBPyBLWEvvgdKHqiVL2R9iXyCmgWYqhgladpfgckOwoCIfawkTHKPnPCW3gH/wJc +/DeV1WIdBM5IFrAGhcgPgUIgYBJkprlaI+Fxm2bltpJJMtYUebmUJQ31OGIfMOKPbIxzDT7klTZq +PF1c5XyTVKiS5tpkJmzxsrBi/fia5w3TAMutiGamaUOnDU4vLdbxXBqXZC5XKAl6kV7bZYcxg54x +yRZXYsNWBt4BWWTCFqRfsaDSWVWSnACAwcIXZ0lRp9RIIYOJGAbaFAR/E6NJz7WzBOzNZjlAhcTm +ewH2B3D7O4jR3ToB+iwAAmgY1FKwfPOkKtFBaPRR4Bt905/HB049W2nbxEOu4iTVVj7OgjN6eFqW +JL4LWWCvqSaGghlmFbp21xnQEcV8NBoFgXGHtsp8zVVQldsjYAVhxpnN5nWChm82Q1Ovf6iARxHO +wF43287CAw1hOn0AKjldVmW+wdd2bp9AmcBY2CPYExekZSQ7yB4nvkbyuSq9ME3RdjvsLFAPBRc/ +nb4/+3L6SRyLy0alTdv67ArGPM1iYGuyCMBUrWEbXQYtUfElqPvEezDvxBRgz6g3ia+Mqxp4F1D/ +XNb0Gqax8F4Gpx9O3pyfzv7y6fSn2aezz6eAINgZGezRlNE81uAwqgiEA7hyqSJtX4NOD3rw5uST +fRDMEjX75mtgN3gyvpYVMHE5hhlPRbiJ7xUwaDilphPEsdMALHg4mYjvxOHz568OCVqxLbYADMyu +0xQfzrRFnyXZKg8n1PgXdumPWUlp/+3y6OsrcXwswl/i2zgMwIdqmjJL/Eji9HlbSOhawZ9xriZB +sJQrEL0biQI6fk5+8YQ7wJJAy1zb6V/yJDPvmSvdIUh/jKkH4DCbLdJYKWw8m4VABOrQ84EOETvX +KHVj6Fhs3a4TjQp+SgkLm2GXKf7Tg2I8p36IBqPodjGNQFw3i1hJbkXTh36zGeqs2WysBwRhJokB +h4vVUChME9RZZQJ+LXEe6rC5ylP8ifBRC5AA4tYKtSQukt46RbdxWks1diYFRByPW2RERZso4kdw +UcZgiZulm0za1DQ8A82AfGkOWrRsUQ4/e+DvgLoymzjc6PHei2mGmP477zQIB3A5Q1T3SrWgsHYU +F6cX4tWLw310Z2DPubTU8ZqjhU6yWtqHK1gtIw+MMPcy8uLSZYV6Fp8e7Ya5iezKdFlhpZe4lJv8 +Vi4BW2RgZ5XFT/QGduYwj0UMqwh6nfwBVqHGb4xxH8qzB2lB3wGotyEoZv3N0u9xMEBmChQRb6yJ +1HrXz6awKPPbBJ2N+Va/BFsJyhItpnFsAmfhPCZDkwgaArzgDCl1J0NQh2XNDivhjSDRXiwbxRoR +uHPU1Ff09SbL77IZ74SPUemOJ5Z1UbA082KDZgn2xHuwQoBkDhu7hmgMBVx+gbK1D8jD9GG6QFna +WwAgMPSKtmsOLLPVoynyrhGHRRiT14KEt5ToL9yaIWirZYjhQKK3kX1gtARCgslZBWdVg2YylDXT +DAZ2SOJz3XnEW1AfQIuKEZjNsYbGjQz9Lo9AOYtzVyk5/dAif/nyhdlGrSm+gojNcdLoQqzIWEbF +FgxrAjrBeGQcrSE2uAPnFsDUSrOm2P8k8oK9MVjPCy3b4AfA7q6qiqODg7u7u0hHF/Ly+kCtDv74 +p2+++dML1onLJfEPTMeRFh1qiw7oHXq00bfGAn1nVq7Fj0nmcyPBGkvyysgVRfy+r5NlLo72J1Z/ +Ihc3Zhr/Na4MKJCZGZSpDLQdNRg9U/vPoldqJJ6RdbZtxxP2S7RJtVbMt7rQo8rBEwC/ZZHXaKob +TlDiK7BusENfynl9HdrBPRtpfsBUUU7Hlgf2X14hBj5nGL4ypniGWoLYAi2+Q/qfmG1i8o60hkDy +oonq7J63/VrMEHf5eHm3vqYjNGaGiULuQInwmzxaAG3jruTgR7u2aPcc19Z8PENgLH1gmFc7lmMU +HMIF12LqSp3D1ejxgjTdsWoGBeOqRlDQ4CTOmdoaHNnIEEGid2M2+7ywugXQqRU5NPEBswrQwh2n +Y+3arOB4QsgDx+IlPZHgIh913r3gpa3TlAI6LR71qMKAvYVGO50DX44NgKkYlX8ZcUuzTfnYWhRe +gx5gOceAkMFWHWbCN64PONob9bBTx+oP9WYa94HARRpzLOpR0AnlYx6hVCBNxdjvOcTilrjdwXZa +HGIqs0wk0mpAuNrKo1eodhqmVZKh7nUWKVqkOXjFVisSIzXvfWeB9kH4uM+YaQnUZGjI4TQ6Jm/P +E8BQt8Pw2XWNgQY3DoMYbRJF1g3JtIZ/wK2g+AYFo4CWBM2CeayU+RP7HWTOzld/GWAPS2hkCLfp +kBvSsRgajnm/J5CMOhoDUpABCbvCSK4jq4MUOMxZIE+44bUclG6CESmQM8eCkJoB3Omlt8HBJxGe +gJCEIuT7SslCfCVGsHxtUX2c7v5dudQEIcZOA3IVdPTi2I1sOFGN41aUw2doP75BZyVFDhw8B5fH +DfS7bG6Y1gZdwFn3FbdFCjQyxWFGExfVK0MYN5j8h2OnRUMsM4hhKG8g70jHjDQJ7HJr0LDgBoy3 +5u2x9GM3YoF9x2GuDuXmHvZ/YZmoRa5Cipm0YxfuR3NFlzYW2/NkPoI/3gKMJlceJJnq+AVGWf6B +QUIPetgH3ZsshkWWcXmXZCEpME2/Y39pOnhYUnpG7uATbacOYKIY8Tx4X4KA0NHnAYgTagLYlctQ +abe/C3bnFEcWLncfeW7z5dGrqy5xp0MRHvvpX6rT+6qMFa5WyovGQoGr1TXgqHRhcnG21YeX+nAb +twllrmAXKT5++iKQEBzXvYu3T5t6w/CIzYNz8j4GddBrD5KrNTtiF0AEtSIyykH4dI58PLJPndyO +iT0ByJMYZseiGEiaT/4ROLsWCsbYX24zjKO1VQZ+4PU3X896IqMukt98PXpglBYx+sR+3PIE7cic +VLBrtqWMU3I1nD4UVMwa1rFtignrc9r+aR676vE5NVo29t3fAj8GCobUJfgIL6YN2bpTxY/vTg3C +03ZqB7DObtV89mgRYG+fz3+BHbLSQbXbOEnpXAEmv7+PytVs7jle0a89PEg7FYxDgr79l7p8AdwQ +cjRh0p2OdsZOTMC5ZxdsPkWsuqjs6RyC5gjMywtwjz+HFU6ve+B7Bge/r7p8IiBvTqMeMmpbbIZ4 +wQclhz1K9gnzfvqMf9dZP27mw4L1/zHLF/+cST5hKgaaNh4+rH5iuXbXAHuEeRpwO3e4hd2h+axy +ZZw7VklKPEfd9VzcUboCxVbxpAigLNnv64GDUqoPvd/WZclH16QCC1nu43HsVGCmlvH8ek3Mnjj4 +ICvExDZbUKzayevJ+4Qv1NFnO5Ow2Tf0c+c6NzErmd0mJfQFhTsOf/j442nYb0IwjgudHm9FHu83 +INwnMG6oiRM+pQ9T6Cld/nH10d66+AQ1GQEmIqzJ1iVsJxBs4gj9a/BARMg7sOVjdtyhL9ZycTOT +lDqAbIpdnaD4W3yNmNiMAj//S8UrSmKDmSzSGmnFjjdmH67qbEHnI5UE/0qnCmPqECUEcPhvlcbX +Ykydlxh60txI0anbuNTeZ1HmmJwq6mR5cJ0shfy1jlPc1svVCnDBwyv9KuLhKQIl3nFOAyctKrmo +y6TaAglileuzP0p/cBrOtzzRsYckH/MwATEh4kh8wmnjeybc0pDLBAf8Ew+cJO67sYOTrBDRc3if +5TMcdUY5vlNGqnsuT4+D9gg5ABgBUJj/aKIjd/4bSa/cA0Zac5eoqCU9UrqRhpycMYQynmCkg3/T +T58RXd4awPJ6GMvr3Vhet7G87sXy2sfyejeWrkjgwtqglZGEvsBV+1ijN9/GjTnxMKfxYs3tMPcT +czwBoijMBtvIFKdAe5EtPt8jIKS2nQNnetjkzyScVFrmHALXIJH78RBLb+ZN8rrTmbJxdGeeinFn +h3KI/L4HUUSpYnPqzvK2jKs48uTiOs3nILYW3WkDYCra6UQcK81uZ3OO7rYs1ejiPz//8PEDNkdQ +I5PeQN1wEdGw4FTGz+PyWnWlqdn8FcCO1NJPxKFuGuDeIyNrPMoe//OOMjyQccQdZSjkogAPgLK6 +bDM39ykMW891kpR+zkzOh03HYpRVo2ZSA0Q6ubh4d/L5ZEQhv9H/jlyBMbT1pcPFx7SwDbr+m9vc +Uhz7gFDr2FZj/Nw5ebRuOOJhG2vAdjzf1oPDxxjs3jCBP8t/KqVgSYBQkQ7+PoVQj945/Kb9UIc+ +hhE7yX/uyRo7K/adI3uOi+KIft+xQ3sA/7AT9xgzIIB2ocZmZ9DslVtK35rXHRR1gD7S1/vNe832 +1qu9k/EpaifR4wA6lLXNht0/75yGjZ6S1ZvT788+nJ+9uTj5/IPjAqIr9/HTwaE4/fGLoPwQNGDs +E8WYGlFhJhIYFrfQSSxz+K/GyM+yrjhIDL3enZ/rk5oNlrpg7jPanAiecxqThcZBM45C24c6/wgx +SvUGyakponQdqjnC/dKG61lUrvOjqVRpjs5qrbdeulbM1JTRuXYE0geNXVIwCE4xg1eUxV6ZXWHJ +J4C6zqoHKW2jbWJISkHBTrqAc/5lTle8QCl1hidNZ63oL0MX1/AqUkWawE7udWhlSXfD9JiGcfRD +e8DNePVpQKc7jKwb8qwHsUCr9Trkuen+k4bRfq0Bw4bB3sG8M0npIZSBjcltIsRGfJITynv4apde +r4GCBcODvgoX0TBdArOPYXMt1glsIIAn12B9cZ8AEFor4R8IHDnRAZljdkb4drPc/3OoCeK3/vnn +nuZVme7/TRSwCxKcShT2ENNt/A42PpGMxOnH95OQkaPUXPHnGssDwCGhAKgj7ZS/xCfos7GS6Urn +l/j6AF9oP4Fet7qXsih1937XOEQJeKbG5DU8U4Z+IaZ7WdhTnMqkBRorHyxmWEHopiGYz574tJZp +qvPdz96dn4LviMUYKEF87nYKw3G8BI/QdfIdVzi2QOEBO7wukY1LdGEpyWIZec16g9YoctTby8uw +60SB4W6vThS4jBPloj3GaTMsU04QISvDWphlZdZutUEKu22I4igzzBKzi5ISWH2eAF6mpzFviWCv +hKUeJgLPp8hJVpmMxTRZgB4FlQsKdQpCgsTFekbivDzjGHheKlMGBQ+LbZlcrys83YDOEZVgYPMf +T76cn32gsoTDV43X3cOcU9oJTDmJ5BhTBDHaAV/ctD/kqtmsj2f1K4SB2gf+tF9xdsoxD9Dpx4FF +/NN+xXVox85OkGcACqou2uKBGwCnW5/cNLLAuNp9MH7cFMAGMx8MxSKx7EUnerjz63KibdkyJRT3 +MS+fcICzKmxKmu7spqS1P3qOqwLPuZbj/kbwtk+2zGcOXW86b4aS39xPRwqxJBYw6rb2xzDZYZ2m +ejoOsw1xC21rtY39OXNipU67RYaiDEQcu50nLpP1K2HdnDnQS6PuABPfanSNJPaq8tHP2Uh7GB4m +ltidfYrpSGUsZAQwkiF17U8NPhRaBFAglP07diR3Onl+6M3RsQYPz1HrLrCNP4Ai1Lm4VOORl8CJ +8OVXdhz5FaGFevRIhI6nkskst3li+Llbo1f50p9jrwxQEBPFroyzazlmWFMD8yuf2AMhWNK2Hqkv +k6s+wyLOwDm9H+Dwrlz0H5wY1FqM0Gl3I7dtdeSTBxv0loLsJJgPvozvQPcXdTXmlRw4h+6tpRuG ++jBEzD6Epvr0fRxiOObXcGB9GsC91NCw0MP7deDsktfGOLLWPraqmkL7QnuwixK2ZpWiYxmnONH4 +otYLaAzucWPyR/apThSyv3vqxJyYkAXKg7sgvbmNdINWOGHE5UpcOZpQOnxTTaPfLeWtTMFogJEd +Y7XDL7baYRLZcEpvHthvxu5ie7Htx43eNJgdmXIMRIAKMXoDPbsQanDAFf5Z70Ti7Iac47d/PZuK +tx9+gn/fyI9gQbHmcSr+BqOLt3kJ20ou2qXbFLCAo+L9Yl4rLIwkaHRCwRdPoLd24ZEXT0N0ZYlf +UmIVpMBk2nLDt50AijxBKmRv3ANTLwG/TUFXywk1DmLfWoz0S6TBcI0L1oUc6JbRutqkaCac4Eiz +iJej87O3px8+nUbVPTK2+Tlygid+HhZORx8Nl3gMNhX2yaLGJ1eOv/yDTIsed1nvNU29DO41RQjb +kcLuL/kmjdjuKeISAwai2C7zRYQtgdO5RK+6A/954mwrH7TvnnFFWOOJPjxrnHh8DNQQP7f1zwga +Uh89J+pJCMVzrBXjx9Go3wJPBUW04c/zm7ulGxDXRT80wTamzazHfnerAtdMZw3PchLhdWyXwdSB +pkmsNvOFWx/4MRP6IhRQbnS8IVdxnVZCZrCVor093UgBCt4t6WMJYVZhK0Z1bhSdSe/irXJyj2Il +RjjqiIrq8RyGAoWw9f4xvmEzgLWGouYSaIBOiNK2KXe6qnqxZgnmnRBRryff4C7JXrnJL5rCPChv +jBeN/wrzRG+RMbqWlZ4/PxhPLl82CQ4UjF54Bb2LAoydyyZ7oDGL58+fj8S/Pez0MCpRmuc34I0B +7F5n5ZxeDxhsPTm7Wl2H3ryJgB8Xa3kJD64oaG6f1xlFJHd0pQWR9q+BEeLahJYZTfuWOeZYXcnn +y9yCz6m0wfhLltB1RxhRkqhs9a1RGG0y0kQsCYohjNUiSUKOTsB6bPMaa/Ewuqj5Rd4DxycIZopv +8WCMd9hrdCwpb9Zyj0XnWIwI8IhSyng0KmamajTAc3ax1WjOzrKkaspIXrhnpvoKgMreYqT5SsR3 +KBlmHi1iOGWdHqs2jnW+k0W9jUq+uHTjjK1Z8uuHcAfWBknLVyuDKTw0i7TIZbkw5hRXLFkklQPG +tEM43JkubyLrEwU9KI1AvZNVWFqJtm//YNfFxfQjHR/vm5F01lBlL8TimFCctfIKo6gZn6JPlpCW +b82XCYzygaLZ2hPwxhJ/0LFUrCHw7u1wyxnrTN/HwWkbzSUdAIfugLIK0rKjpyOci8csfGbagVs0 +8EM7c8LtNimrOk5n+tqHGfppM3uervG0ZXA7CzyttwK+fQ6O777O2AfHwSTXID0x49ZUZByLlY5M +RG5lmV+EVeTo5R2yrwQ+BVJmOTP10CZ2dGnZ1Raa6gRHR8UjqK9M8dKAQ26qZjoFJy7mU0pvMuUO +A86zn29JV1eI78T41VQctnY+i2KLNzkBss+Woe+KUTeYihMMMHNs34shvjsW45dT8ccd0KOBAY4O +3RHa+9gWhEEgr66eTMY0mRPZwr4U9of76hxG0PSM4+SqTf4umb4lKv1ri0pcIagTlV+2E5VbYw/u +WzsfH8lwA4pjlcjl/jOFJNRIN7p5mMEJPyyg37M5Wrp2vKmoocK5OWxG7ho96GhE4zbbQUxRulZf +XL+LuoYNp71zwKTJtFIV7S1zmMao0WsRFQDM+o7S8Bve7QLvNSlc/2zwiFUXAViwPREEXenJB2ZN +w0ZQH3QEn6QBHmAUEeJhaqMoXMl6goiEdA8OMdFXrUNsh+N/d+bhEoOho9AOlt98vQtPVzB7izp6 +FnR3pYUnsra8ollu8+kPzHmM0tf1NwmMA6URHXBWzVWV5GYeYfYy30GT2yzmDV4GSSfTaBJT6bpN +vJXmW7/Qj6HYASWTwVqAJ1Wv8CD5lu62PFGU9IZX1Hx9+HJqKoMZkJ7Aq+jVV/oKSOpmLj/wfeyp +3rvBS93vMPoXB1hS+b3tq85uhqZ13LoLyh8spOjZJJpZOjSG6eE6kGbNYoF3JjbEZN/aXgDyHryd +Ofg55vLTHBw22JBGfei6GqOR3iHVNiDAD5uMIcl5VNdGkSLSu4RtSHnuUpxPFgXdq9+CYAgBOX8d +8xt0BeviyIbYjE3Bk8+xm82Jn+qmt+6M7Qka2+om3DV97r9r7rpFYGdukhk6c/frS10a6L7DVrSP +Bhze0IR4VIlEo/H7jYlrB6Y6h6Y/Qq8/SH63E850wKw8BMZk7GC8n9hTY2/M/iZeuN8xIWyfL2R2 +y4l7nY3WtDs2o83xj/EUOPkFn9sbBiijaak5kPdLdMPejHNkZ/L6Ws1ivN1xRptsyufq7J7Mtu09 +Xc4nY7U1uy28tAhAGG7Smbducj0wBuhKvmWa06Gc22kEDU1Jw04WskqWbBL01g7ARRwxpf4mEM9p +xKNUYqBb1WVRwm54pO8i5jydvtTmBqgJ4G1idWNQNz2m+mpaUqyUHGZKkDlO20ryASKwEe+YhtnM +vgNeedFcs5BMLTPIrN7IMq6aK4b8jIAENl3NCFR0jovrhOcaqWxxiYtYYnnDQQoDZPb7V7Cx9DbV +O+5VmFht93h2oh465PuUKxscY2S4OLm31wu611ot6Wpr1zu0zRqus1cqwTKYu/JIR+pYGb/V93fx +HbMcyUf/0uEfkHe38tLPQrfqjL1bi4bzzFUI3Qub8MYAMs599zB2OKB742JrA2zH9/WFZZSOhznQ +2FJR++S9CqcZbdJEkDBh9IEIkl8U8MQIkgf/kREkfWsmGBqNj9YDvWUCD4SaWD24V1A2jAB9ZkAk +PMBuXWBoTOXYTbovcpXcj+yF0qwrnUo+Yx6QI7t3kxEIvmpSuRnK3lVwuyJIvnTR4+/PP745OSda +zC5O3v7HyfeUlIXHJS1b9egQW5bvM7X3vfRvN9ymE2n6Bm+w7bkhlmuYNITO+04OQg+E/nq1vgVt +KzL39VCHTt1PtxMgvnvaLahDKrsXcscv0zUmbvpMK0870E85qdb8cjITzCNzUsfi0JzEmffN4YmW +0U5seWjhnPTWrjrR/qq+BXQg7j2xSda0Anhmgvxlj0xMxYwNzLOD0v7ffFBmOFYbmht0QAoX0rnJ +kS5xZFCV//8TKUHZxbi3Y0dxau/mpnZ8PKTspfN49ruQkSGIV+436s7PFfalTAeoEASs8PQ9hYyI +0X/6QNWmHzxT4nKfCov3Udlc2V+4Ztq5/WuCSQaVve9LcYISH7NC41WduokDtk+nAzl9dBqVr5xK +FtB8B0DnRjwVsDf6S6wQ51sRwsZRu2SYHEt01Jf1Ocij3XSwN7R6IfaHyk7dskshXg43XLYqO3WP +Q+6hHuihalPc51hgzNIcqicV3xFkPs4UdMGX53zgGbre9sPX28uXR/ZwAfkdXzuKhLLJRo5hv3Sy +MXdeKul0J2Ypp5Suh3s1JySsW1w5UNknGNrbdEpSBvY/Js+BIY289/0hM9PDu3p/1MbUst4RTEmM +n6kJTcsp4tG42yeT7nQbtdUFwgVJjwDSUYEAC8F0dKOTILrlLO/xC70bnNd0Ha97whQ6UkHJYj5H +cA/j+zX4tbtTIfGjujOKpj83aHOgXnIQbvYduNXEC4UMm4T21Bs+GHABuCa7v//LR/TvpjHa7oe7 +/Grb6lVvHSD7spj5iplBLRKZxxEYGdCbY9LWWC5hBB2voWno6DJUMzfkC3T8KJsWL9umDQY5szPt +AVijEPwfucjncQ== +""") + +##file activate.sh +ACTIVATE_SH = convert(""" +eJytVd9v2kAMfs9fYQLq2m4MscdNVKMqEkgtVIQxbeuUHolpTgsXdHehpT/+9/mSEBJS2MOaB0ji +z77P9menDpOAK5jzEGERKw0zhFihD/dcB2CrKJYewoyLFvM0XzGNNpzOZbSAGVPBqVWHdRSDx4SI +NMhYANfgc4meDteW5ePGC45P4MkCumKhUENzDsu1H3lw1vJx1RJxGMKns6O2lWDqINGgotAHFCsu +I7FAoWHFJGezEFWGqsEvaD5C42naHb93X+A3+elYCgVaxgh8DmQAys9HL2SS0mIaWBgm7mTN/O3G +kzu6vHCng/HkW/fSve5O+hTOpnhfQAcoEry5jKVjNypoO0fgwzKSOgHm79KUK06Jfc7/RebHpD8a +9kdXvT2UcnuFWG6p0stNB0mWUUQ1q3uiGRVEMfXHR03dTuQATPjwqIIPcB9wL4CArRAY/ZHJixYL +Y9YBtcAoLQtFevOoI9QaHcEdMSAB0d08kuZhyUiSmav6CPCdVBnFOjNrLu6yMCWgKRA0TInBC5i4 +QwX3JG/mm581GKnSsSSxJTFHf9MAKr8w5T/vOv1mUurn5/zlT6fvTntjZzAaNl9rQ5JkU5KIc0GX +inagwU57T2eddqWlTrvaS6d9sImZeUMkhWysveF0m37NcGub9Dpgi0j4qGiOzATjDr06OBjOYQOo +7RBoGtNm9Denv1i0LVI7lxJDXLHSSBeWRflsyyqw7diuW3h0XdvK6lBMyaoMG1UyHdTsoYBuue75 +YOgOu1c91/2cwYpznPPeDoQpGL2xSm09NKp7BsvQ2hnT3aMs07lUnskpxewvBk73/LLnXo9HV9eT +ijB3hWBO2ygoiWg/bKuZxqCCQq0DD3vkWIVvI2KosIw+vqW1gIItEG5KJb+xb09g65ktwYKgTc51 +uGJ/EFQs0ayEWLCQM5V9N4g+1+8UbXOJzF8bqhKtIqIwicWvzNFROZJlpfD8A7Vc044R0FxkcezG +VzsV75usvTdYef+57v5n1b225qhXfwEmxHEs +""") + +##file activate.fish +ACTIVATE_FISH = convert(""" +eJyFVVFv0zAQfs+vONJO3RDNxCsSQoMVrdK2Vl03CSHkesllMXLsYDvZivjx2GmTOG0YfWhV+7u7 +73z33Y1gnTENKeMIeakNPCKUGhP7xcQTbCJ4ZOKcxoZV1GCUMp1t4O0zMxkTQEGVQjicO4dTyIwp +Ppyfu386Q86jWOZwBhq1ZlK8jYIRXEoQ0jhDYAYSpjA2fBsFQVoKG0UKSLAJB9MEJrMXi6uYMiXl +KCrIZYJARQIKTakEGAkmQ+tU5ZSDRTAlRY7CRJMA7GdkgRoNSJ74t1BRxegjR12jWAoGbfpTAeGY +LK4vycN8tb6/uCbLi/VVWGPcx3maPr2AO4VjYB+HMAxAkQT/i/ptfbW4vVrczAZit3eHDNqL13n0 +Ya+w+Tq/uyLL1eJmuSaLh9lqNb/0+IzgznqnAjAvzBa4jG0BNmNXfdJUkxTU2I6xRaKcy+e6VApz +WVmoTGFTgwslrYdN03ONrbbMN1E/FQ7H7gOP0UxRjV67TPRBjF3naCMV1mSkYk9MUN7F8cODZzsE +iIHYviIe6n8WeGQxWKuhl+9Xa49uijq7fehXMRxT9VR9f/8jhDcfYSKkSOyxKp22cNIrIk+nzd2b +Yc7FNpHx8FUn15ZfzXEE98JxZEohx4r6kosCT+R9ZkHQtLmXGYSEeH8JCTvYkcRgXAutp9Rw7Jmf +E/J5fktuL25m1tMe3vLdjDt9bNxr2sMo2P3C9BccqGeYhqfQITz6XurXaqdf99LF1mT2YJrvzqCu +5w7dKvV3PzNyOb+7+Hw923dOuB+AX2SxrZs9Lm0xbCH6kmhjUyuWw+7cC7DX8367H3VzDz6oBtty +tMIeobE21JT6HaRS+TbaoqhbE7rgdGs3xtE4cOF3xo0TfxwsdyRlhUoxuzes18r+Jp88zDx1G+kd +/HTrr1BY2CeuyfnbQtAcu9j+pOw6cy9X0k3IuoyKCZPC5ESf6MkgHE5tLiSW3Oa+W2NnrQfkGv/h +7tR5PNFnMBlw4B9NJTxnzKA9fLTT0aXSb5vw7FUKzcTZPddqYHi2T9/axJmEEN3qHncVCuEPaFmq +uEtpcBj2Z1wjrqGReJBHrY6/go21NA== +""") + +##file activate.csh +ACTIVATE_CSH = convert(""" +eJx1U2FP2zAQ/e5f8TAV3Soo+0zXbYUiDQkKQgVp2ibjJNfFUuIg22nVf885SVFLO3+I7Lt3fr6X +d8eY58ZjYQpCWfuAhFB7yrAyIYf0Ve1SQmLsuU6DWepAw9TnEoOFq0rwdjAUx/hV1Ui1tVWAqy1M +QGYcpaFYx+yVI67LkKwx1UuTEaYGl4X2Bl+zJpAlP/6V2hTDtCq/DYXQhdEeGW040Q/Eb+t9V/e3 +U/V88zh/mtyqh8n8J47G+IKTE3gKZJdoYrK3h5MRU1tGYS83gqNc+3yEgyyP93cP820evHLvr2H8 +kaYB/peoyY7aVHzpJnE9e+6I5Z+ji4GMTNJWNuOQq6MA1N25p8pW9HWdVWlfsNpPDbdxjgpaahuw +1M7opCA/FFu1uwxC7L8KUqmto1KyQe3rx0I0Eovdf7BVe67U5c1MzSZ310pddGheZoFPWyytRkzU +aCA/I+RkBXhFXr5aWV0SxjhUI6jwdAj8kmhPzX7nTfJFkM3MImp2VdVFFq1vLHSU5szYQK4Ri+Jd +xlW2JBtOGcyYVW7SnB3v6RS91g3gKapZ0oWxbHVteYIIq3iv7QeuSrUj6KSqQ+yqsxDj1ivNQxKF +YON10Q+NH/ARS95i5Tuqq2Vxfvc23f/FO6zrtXXmJr+ZtMY9/A15ZXFWtmch2rEQ4g1ryVHH +""") + +##file activate.bat +ACTIVATE_BAT = convert(""" +eJx9Ul9LhEAQfxf8DoOclI/dYyFkaCmcq4gZQTBUrincuZFbff12T133TM+nnd35/Zvxlr7XDFhV +mUZHOVhFlOWP3g4DUriIWoVomYZpNBWUtGpaWgImO191pFkSpzlcmgaI70jVX7n2Qp8tuByg+46O +CMHbMq64T+nmlJt082D1T44muCDk2prgEHF4mdI9RaS/QwSt3zSyIAaftRccvqVTBziD1x/WlPD5 +xd729NDBb8Nr4DU9QNMKsJeH9pkhPedhQsIkDuCDCa6A+NF9IevVFAohkqizdHetg/tkWvPoftWJ +MCqnOxv7/x7Np6yv9P2Ker5dmX8yNyCkkWnbZy3N5LarczlqL8htx2EM9rQ/2H5BvIsIEi8OEG8U ++g8CsNTr +""") + +##file deactivate.bat +DEACTIVATE_BAT = convert(""" +eJyFkN0KgkAUhO8F32EQpHqFQEjQUPAPMaErqVxzId3IrV6/XST/UDx3c86c4WMO5FYysKJQFVVp +CEfqxsnJ9DI7SA25i20fFqs3HO+GYLsDZ7h8GM3xfLHrg1QNvpSX4CWpQGvokZk4uqrQAjXjyElB +a5IjCz0r+2dHcehHCe5MZNmB5R7TdqMqECMptHZh6DN/utb7Zs6Cej8OXYE5J04YOKFvD4GkHuJ0 +pilSd1jG6n87tDZ+BUwUOepI6CGSkFMYWf0ihvT33Qj1A+tCkSI= +""") + +##file activate.ps1 +ACTIVATE_PS = convert(""" +eJylWdmO41hyfW+g/0FTU7C7IXeJIqmtB/3AnZRIStxF2kaBm7gv4ipyMF/mB3+Sf8GXVGVl1tLT +43ECSqR4b5wbETeWE8z/+a///vNCDaN6cYtSf5G1dbNw/IVXNIu6aCvX9xa3qsgWl0IJ/7IYinbh +2nkOVqs2X0TNjz/8eeFFle826fBhQRaLBkD9uviw+LCy3Sbq7Mb/UNbrH3+YNtLcVaB+Xbipb+eL +tly0eVsD/M6u6g8//vC+dquobH5VWU75eMFUdvHb4n02RHlXuHYTFfmHbHCLLLNz70NpN+GrBI4p +1EeSk4FAXaZR88u0vPip8usi7fznt3fvP+OuPnx49/Pil4td+XnzigIAPoqYQH2J8v4z+C+8b98m +Q25t7k76LIK0cOz0V89/MXXx0+Lf6z5q3PA/F+/FIif9uqnaadFf/PzXSXYBfqIb2NeApecJwPzI +dlL/149nnvyoc7KqYfzTAT8v/voUmX7e+3n364tffl/oVaDyswKY/7J18e6bve8Wv9RuUfqfLHmK +/u139Hwx+9ePRep97KKqae30YwmCo2y+0vTz1k+rv7159B3pb1SOGj97Pe8/flfkC1Vn/7xYR4n6 +lypNEGDDV5f7lcjil3S+4++p881Wv6qKyn5GQg1yJwcp4BZ5E+Wt/z1P/umbiHir4J8Xip/eFt6n +9T/9gU9eY+7zUX97Jlmb136ziKrKT/3OzpvP8VX/+MObSP0lL3LvVZlJ9v1b8357jXyw8rXxYPXN +11n4UzJ8G8S/vUbuJ6RPj999DbtS5kys//JusXwrNLnvT99cFlBNwXCe+niRz8JF/ezNr9Pze+H6 +18W7d5PPvozW7+387Zto/v4pL8BvbxTzvIW9KCv/Fj0WzVQb/YXbVlPZWTz3/9vCaRtQbPN/Bb+j +2rUrDxTVD68gfQXu/ZewAFX53U/vf/rD2P3558W7+W79Po1y/xXoX/6RFHyNIoVjgAG4H0RTcAe5 +3bSVv3DSwk2mZYHjFB8zj6fC4sLOFTHJJQrwzFYJgso0ApOoBzFiRzzQKjIQCCbQMIFJGCKqGUyS +8AkjiF2wTwmMEbcEUvq8Nj+X0f4YcCQmYRiOY7eRbAJDqzm1chOoNstbJ8oTBhZQ2NcfgaB6QjLp +U4+SWFjQGCZpyqby8V4JkPGs9eH1BscXIrTG24QxXLIgCLYNsIlxSYLA6SjAeg7HAg4/kpiIB8k9 +TCLm0EM4gKIxEj8IUj2dQeqSxEwYVH88qiRlCLjEYGuNIkJB1BA5dHOZdGAoUFk54WOqEojkuf4Q +Ig3WY+96TDlKLicMC04h0+gDCdYHj0kz2xBDj9ECDU5zJ0tba6RKgXBneewhBG/xJ5m5FX+WSzsn +wnHvKhcOciw9NunZ0BUF0n0IJAcJMdcLqgQb0zP19dl8t9PzmMBjkuIF7KkvHgqEovUPOsY0PBB1 +HCtUUhch83qEJPjQcNQDsgj0cRqx2ZbnnlrlUjE1EX2wFJyyDa/0GLrmKDEFepdWlsbmVU45Wiwt +eFM6mfs4kxg8yc4YmKDy67dniLV5FUeO5AKNPZaOQQ++gh+dXE7dbJ1aTDr7S4WPd8sQoQkDyODg +XnEu/voeKRAXZxB/e2xaJ4LTFLPYEJ15Ltb87I45l+P6OGFA5F5Ix8A4ORV6M1NH1uMuZMnmFtLi +VpYed+gSq9JDBoHc05J4OhKetrk1p0LYiKipxLMe3tYS7c5V7O1KcPU8BJGdLfcswhoFCSGQqJ8f +ThyQKy5EWFtHVuNhvTnkeTc8JMpN5li3buURh0+3ZGuzdwM55kon+8urbintjdQJf9U1D0ah+hNh +i1XNu4fSKbTC5AikGEaj0CYM1dpuli7EoqUt7929f1plxGGNZnixFSFP2qzhlZMonu2bB9OWSqYx +VuHKWNGJI8kqUhMTRtk0vJ5ycZ60JlodlmN3D9XiEj/cG2lSt+WV3OtMgt1Tf4/Z+1BaCus740kx +Nvj78+jMd9tq537Xz/mNFyiHb0HdwHytJ3uQUzKkYhK7wjGtx3oKX43YeYoJVtqDSrCnQFzMemCS +2bPSvP+M4yZFi/iZhAjL4UOeMfa7Ex8HKBqw4umOCPh+imOP6yVTwG2MplB+wtg97olEtykNZ6wg +FJBNXSTJ3g0CCTEEMdUjjcaBDjhJ9fyINXgQVHhA0bjk9lhhhhOGzcqQSxYdj3iIN2xGEOODx4qj +Q2xikJudC1ujCVOtiRwhga5nPdhe1gSa649bLJ0wCuLMcEYIeSy25YcDQHJb95nfowv3rQnin0fE +zIXFkM/EwSGxvCCMgEPNcDp/wph1gMEa8Xd1qAWOwWZ/KhjlqzgisBpDDDXz9Cmov46GYBKHC4zZ +84HJnXoTxyWNBbXV4LK/r+OEwSN45zBp7Cub3gIYIvYlxon5BzDgtPUYfXAMPbENGrI+YVGSeTQ5 +i8NMB5UCcC+YRGIBhgs0xhAGwSgYwywpbu4vpCSTdEKrsy8osXMUnHQYenQHbOBofLCNNTg3CRRj +A1nXY2MZcjnXI+oQ2Zk+561H4CqoW61tbPKv65Y7fqc3TDUF9CA3F3gM0e0JQ0TPADJFJXVzphpr +2FzwAY8apGCju1QGOiUVO5KV6/hKbtgVN6hRVwpRYtu+/OC6w2bCcGzZQ8NCc4WejNEjFxOIgR3o +QqR1ZK0IaUxZ9nbL7GWJIjxBARUhAMnYrq/S0tVOjzlOSYRqeIZxaSaOBX5HSR3MFekOXVdUPbjX +nru61fDwI8HRYPUS7a6Inzq9JLjokU6P6OzT4UCH+Nha+JrU4VqEo4rRHQJhVuulAnvFhYz5NWFT +aS/bKxW6J3e46y4PLagGrCDKcq5B9EmP+s1QMCaxHNeM7deGEV3WPn3CeKjndlygdPyoIcNaL3dd +bdqPs47frcZ3aNWQ2Tk+rjFR01Ul4XnQQB6CSKA+cZusD0CP3F2Ph0e78baybgioepG12luSpFXi +bHbI6rGLDsGEodMObDG7uyxfCeU+1OiyXYk8fnGu0SpbpRoEuWdSUlNi5bd9nBxYqZGrq7Qa7zV+ +VLazLcelzzP9+n6+xUtWx9OVJZW3gk92XGGkstTJ/LreFVFF2feLpXGGuQqq6/1QbWPyhJXIXIMs +7ySVlzMYqoPmnmrobbeauMIxrCr3sM+qs5HpwmmFt7SM3aRNQWpCrmeAXY28EJ9uc966urGKBL9H +18MtDE5OX97GDOHxam11y5LCAzcwtkUu8wqWI1dWgHyxGZdY8mC3lXzbzncLZ2bIUxTD2yW7l9eY +gBUo7uj02ZI3ydUViL7oAVFag37JsjYG8o4Csc5R7SeONGF8yZP+7xxi9scnHvHPcogJ44VH/LMc +Yu6Vn3jEzCFw9Eqq1ENQAW8aqbUwSiAqi+nZ+OkZJKpBL66Bj8z+ATqb/8qDIJUeNRTwrI0YrVmb +9FArKVEbCWUNSi8ipfVv+STgkpSsUhcBg541eeKLoBpLGaiHTNoK0r4nn3tZqrcIULtq20Df+FVQ +Sa0MnWxTugMuzD410sQygF4qdntbswiJMqjs014Irz/tm+pd5oygJ0fcdNbMg165Pqi7EkYGAXcB +dwxioCDA3+BY9+JjuOmJu/xyX2GJtaKSQcOZxyqFzTaa6/ot21sez0BtKjirROKRm2zuai02L0N+ +ULaX8H5P6VwsGPbYOY7sAy5FHBROMrMzFVPYhFHZ7M3ZCZa2hsT4jGow6TGtG8Nje9405uMUjdF4 +PtKQjw6yZOmPUmO8LjFWS4aPCfE011N+l3EdYq09O3iQJ9a01B3KXiMF1WmtZ+l1gmyJ/ibAHZil +vQzdOl6g9PoSJ4TM4ghTnTndEVMOmsSSu+SCVlGCOLQRaw9oLzamSWP62VuxPZ77mZYdfTRGuNBi +KyhZL32S2YckO/tU7y4Bf+QKKibQSKCTDWPUwWaE8yCBeL5FjpbQuAlb53mGX1jptLeRotREbx96 +gnicYz0496dYauCjpTCA4VA0cdLJewzRmZeTwuXWD0talJsSF9J1Pe72nkaHSpULgNeK1+o+9yi0 +YpYwXZyvaZatK2eL0U0ZY6ekZkFPdC8JTF4Yo1ytawNfepqUKEhwznp6HO6+2l7L2R9Q3N49JMIe +Z+ax1mVaWussz98QbNTRPo1xu4W33LJpd9H14dd66ype7UktfEDi3oUTccJ4nODjwBKFxS7lYWiq +XoHu/b7ZVcK5TbRD0F/2GShg2ywwUl07k4LLqhofKxFBNd1grWY+Zt/cPtacBpV9ys2z1moMLrT3 +W0Elrjtt5y/dvDQYtObYS97pqj0eqmwvD3jCPRqamGthLiF0XkgB6IdHLBBwDGPiIDh7oPaRmTrN +tYA/yQKFxRiok+jM6ciJq/ZgiOi5+W4DEmufPEubeSuYJaM3/JHEevM08yJAXUQwb9LS2+8FOfds +FfOe3Bel6EDSjIEIKs4o9tyt67L1ylQlzhe0Q+7ue/bJnWMcD3q6wDSIQi8ThnRM65aqLWesi/ZM +xhHmQvfKBbWcC194IPjbBLYR9JTPITbzwRcu+OSFHDHNSYCLt29sAHO6Gf0h/2UO9Xwvhrjhczyx +Ygz6CqP4IwxQj5694Q1Pe2IR+KF/yy+5PvCL/vgwv5mPp9n4kx7fnY/nmV++410qF/ZVCMyv5nAP +pkeOSce53yJ6ahF4aMJi52by1HcCj9mDT5i+7TF6RoPaLL+cN1hXem2DmX/mdIbeeqwQOLD5lKO/ +6FM4x77w6D5wMx3g0IAfa2D/pgY9a7bFQbinLDPz5dZi9ATIrd0cB5xfC0BfCCZO7TKP0jQ2Meih +nRXhkA3smTAnDN9IW2vA++lsgNuZ2QP0UhqyjUPrDmgfWP2bWWiKA+YiEK7xou8cY0+d3/bk0oHR +QLrq4KzDYF/ljQDmNhBHtkVNuoDey6TTeaD3SHO/Bf4d3IwGdqQp6FuhmwFbmbQBssDXVKDBYOpk +Jy7wxOaSRwr0rDmGbsFdCM+7XU/84JPu3D/gW7QXgzlvbjixn99/8CpWFUQWHFEz/RyXvzNXTTOd +OXLNNFc957Jn/YikNzEpUdRNxXcC6b76ccTwMGoKj5X7c7TvHFgc3Tf4892+5A+iR+D8OaaE6ACe +gdgHcyCoPm/xiDCWP+OZRjpzfj5/2u0i4qQfmIEOsTV9Hw6jZ3Agnh6hiwjDtGYxWvt5TiWEuabN +77YCyRXwO8P8wdzG/8489KwfFBZWI6Vvx76gmlOc03JI1HEfXYZEL4sNFQ3+bqf7e2hdSWQknwKF +ICJjGyDs3fdmnnxubKXebpQYLjPgEt9GTzKkUgTvOoQa1J7N3nv4sR6uvYFLhkXZ+pbCoU3K9bfq +gF7W82tNutRRZExad+k4GYYsCfmEbvizS4jsRr3fdzqjEthpEwm7pmN7OgVzRbrktjrFw1lc0vM8 +V7dyTJ71qlsd7v3KhmHzeJB35pqEOk2pEe5uPeCToNkmedmxcKbIj+MZzjFSsvCmimaMQB1uJJKa ++hoWUi7aEFLvIxKxJavqpggXBIk2hr0608dIgnfG5ZEprqmH0b0YSy6jVXTCuIB+WER4d5BPVy9Q +M4taX0RIlDYxQ2CjBuq78AAcHQf5qoKP8BXHnDnd/+ed5fS+csL4g3eWqECaL+8suy9r8hx7c+4L +EegEWdqAWN1w1NezP34xsxLkvRRI0DRzKOg0U+BKfQY128YlYsbwSczEg2LqKxRmcgiwHdhc9MQJ +IwKQHlgBejWeMGDYYxTOQUiJOmIjJbzIzHH6lAMP+y/fR0v1g4wx4St8fcqTt3gz5wc+xXFZZ3qI +JpXI5iJk7xmNL2tYsDpcqu0375Snd5EKsIvg8u5szTOyZ4v06Ny2TZXRpHUSinh4IFp8Eoi7GINJ +02lPJnS/9jSxolJwp2slPMIEbjleWw3eec4XaetyEnSSqTPRZ9fVA0cPXMqzrPYQQyrRux3LaAh1 +wujbgcObg1nt4iiJ5IMbc/WNPc280I2T4nTkdwG8H6iS5xO2WfsFsruBwf2QkgZlb6w7om2G65Lr +r2Gl4dk63F8rCEHoUJ3fW+pU2Srjlmcbp+JXY3DMifEI22HcHAvT7zzXiMTr7VbUR5a2lZtJkk4k +1heZZFdru8ucCWMTr3Z4eNnjLm7LW7rcN7QjMpxrsCzjxndeyFUX7deIs3PQkgyH8k6luI0uUyLr +va47TBjM4JmNHFzGPcP6BV6cYgQy8VQYZe5GmzZHMxyBYhGiUdekZQ/qwyxC3WGylQGdUpSf9ZCP +a7qPdJd31fPRC0TOgzupO7nLuBGr2A02yuUQwt2KQG31sW8Gd9tQiHq+hPDt4OzJuY4pS8XRsepY +tsd7dVEfJFmc15IYqwHverrpWyS1rFZibDPW1hUUb+85CGUzSBSTK8hpvee/ZxonW51TUXekMy3L +uy25tMTg4mqbSLQQJ+skiQu2toIfBFYrOWql+EQipgfT15P1aq6FDK3xgSjIGWde0BPftYchDTdM +i4QdudHFkN0u6fSKiT09QLv2mtSblt5nNzBR6UReePNs+khE4rHcXuoK21igUKHl1c3MXMgPu7y8 +rKQDxR6N/rffXv+lROXet/9Q+l9I4D1U +""") + +##file distutils-init.py +DISTUTILS_INIT = convert(""" +eJytV1uL4zYUfvevOE0ottuMW9q3gVDa3aUMXXbLMlDKMBiNrSTqOJKRlMxkf33PkXyRbGe7Dw2E +UXTu37lpxLFV2oIyifAncxmOL0xLIfcG+gv80x9VW6maw7o/CANSWWBwFtqeWMPlGY6qPjV8A0bB +C4eKSTgZ5LRgFeyErMEeOBhbN+Ipgeizhjtnhkn7DdyjuNLPoCS0l/ayQTG0djwZC08cLXozeMss +aG5EzQ0IScpnWtHSTXuxByV/QCmxE7y+eS0uxWeoheaVVfqSJHiU7Mhhi6gULbOHorshkrEnKxpT +0n3A8Y8SMpuwZx6aoix3ouFlmW8gHRSkeSJ2g7hU+kiHLDaQw3bmRDaTGfTnty7gPm0FHbIBg9U9 +oh1kZzAFLaue2R6htPCtAda2nGlDSUJ4PZBgCJBGVcwKTAMz/vJiLD+Oin5Z5QlvDPdulC6EsiyE +NFzb7McNTKJzbJqzphx92VKRFY1idenzmq3K0emRcbWBD0ryqc4NZGmKOOOX9Pz5x+/l27tP797c +f/z0d+4NruGNai8uAM0bfsYaw8itFk8ny41jsfpyO+BWlpqfhcG4yxLdi/0tQqoT4a8Vby382mt8 +p7XSo7aWGdPBc+b6utaBmCQ7rQKQoWtAuthQCiold2KfJIPTT8xwg9blPumc+YDZC/wYGdAyHpJk +vUbHbHWAp5No6pK/WhhLEWrFjUwtPEv1Agf8YmnsuXUQYkeZoHm8ogP16gt2uHoxcEMdf2C6pmbw +hUMsWGhanboh4IzzmsIpWs134jVPqD/c74bZHdY69UKKSn/+KfVhxLgUlToemayLMYQOqfEC61bh +cbhwaqoGUzIyZRFHPmau5juaWqwRn3mpWmoEA5nhzS5gog/5jbcFQqOZvmBasZtwYlG93k5GEiyw +buHhMWLjDarEGpMGB2LFs5nIJkhp/nUmZneFaRth++lieJtHepIvKgx6PJqIlD9X2j6pG1i9x3pZ +5bHuCPFiirGHeO7McvoXkz786GaKVzC9DSpnOxJdc4xm6NSVq7lNEnKdVlnpu9BNYoKX2Iq3wvgh +gGEUM66kK6j4NiyoneuPLSwaCWDxczgaolEWpiMyDVDb7dNuLAbriL8ig8mmeju31oNvQdpnvEPC +1vAXbWacGRVrGt/uXN/gU0CDDwgooKRrHfTBb1/s9lYZ8ZqOBU0yLvpuP6+K9hLFsvIjeNhBi0KL +MlOuWRn3FRwx5oHXjl0YImUx0+gLzjGchrgzca026ETmYJzPD+IpuKzNi8AFn048Thd63OdD86M6 +84zE8yQm0VqXdbbgvub2pKVnS76icBGdeTHHXTKspUmr4NYo/furFLKiMdQzFjHJNcdAnMhltBJK +0/IKX3DVFqvPJ2dLE7bDBkH0l/PJ29074+F0CsGYOxsb7U3myTUncYfXqnLLfa6sJybX4g+hmcjO +kMRBfA1JellfRRKJcyRpxdS4rIl6FdmQCWjo/o9Qz7yKffoP4JHjOvABcRn4CZIT2RH4jnxmfpVG +qgLaAvQBNfuO6X0/Ux02nb4FKx3vgP+XnkX0QW9pLy/NsXgdN24dD3LxO2Nwil7Zlc1dqtP3d7/h +kzp1/+7hGBuY4pk0XD/0Ao/oTe/XGrfyM773aB7iUhgkpy+dwAMalxMP0DrBcsVw/6p25+/hobP9 +GBknrWExDhLJ1bwt1NcCNblaFbMKCyvmX0PeRaQ= +""") + +##file distutils.cfg +DISTUTILS_CFG = convert(""" +eJxNj00KwkAMhfc9xYNuxe4Ft57AjYiUtDO1wXSmNJnK3N5pdSEEAu8nH6lxHVlRhtDHMPATA4uH +xJ4EFmGbvfJiicSHFRzUSISMY6hq3GLCRLnIvSTnEefN0FIjw5tF0Hkk9Q5dRunBsVoyFi24aaLg +9FDOlL0FPGluf4QjcInLlxd6f6rqkgPu/5nHLg0cXCscXoozRrP51DRT3j9QNl99AP53T2Q= +""") + +##file activate_this.py +ACTIVATE_THIS = convert(""" +eJyNU01v2zAMvetXEB4K21jnDOstQA4dMGCHbeihlyEIDMWmE62yJEiKE//7kXKdpEWLzYBt8evx +kRSzLPs6wiEoswM8YdMpjUXcq1Dz6RZa1cSiTkJdr86GsoTRHuCotBayiWqQEYGtMCgfD1KjGYBe +5a3p0cRKiEe2NtLAFikftnDco0ko/SFEVgEZ8aRCZDIPY9xbA8pE9M4jfW/B2CjiHq9zbJVZuOQq +siwTIvpxKYCembPAU4Muwi/Z4zfvrZ/MXipKeB8C+qisSZYiWfjJfs+0/MFMdWn1hJcO5U7G/SLa +xVx8zU6VG/PXLXvfsyyzUqjeWR8hjGE+2iCE1W1tQ82hsCJN9dzKaoexyB/uH79TnjwvxcW0ntSb +yZ8jq1Z5Q1UXsyy3gf9nbjTEj7NzQMfCJa/YSmrQ+2D/BqfiOi6sclrGzvoeVivIj8rcfcmnIQRF +7XCyeZI7DFe5/lhlCs5PRf5QW66VXT/NrlQ46oD/D6InkOmi3IQcbhKxAX2g4a+Xd5s3UtCtG2py +m8eg6WYWqR6SL5OjKMGfSrYt/6kxxQtOpeAgj1LXBNmpE2ElmCSIy5H0zFd8gJ924HWijWhb2hRC +6wNEm1QdDZtuSZcEprIUBo/XRNcbQe1OUbQ/r3hPTaPJJDNtFLu8KHV5XoNr3Eo6h6YtOKw8e8yw +VF5PnJ+ts3a9/Mz38RpG/AUSzYUW +""") + +##file python-config +PYTHON_CONFIG = convert(""" +eJyNVV1P2zAUfc+v8ODBiSABxlulTipbO6p1LWqBgVhlhcZpPYUkctzSivHfd6+dpGloGH2Ja/ue +e+65Hz78xNhtf3x90xmw7vCWsRPGLvpDNuz87MKfdKMWSWxZ4ilNpCLZJiuWc66SVFUOZkkcirll +rfxIBAzOMtImDzSVPBRrekwoX/OZu/0r4lm0DHiG60g86u8sjPw5rCyy86NRkB8QuuBRSqfAKESn +3orLTCQxE3GYkC9tYp8fk89OSwNsmXgizrhUtnumeSgeo5GbLUMk49Rv+2nK48Cm/qMwfp333J2/ +dVcAGE0CIQHBsgIeEr4Wij0LtWDLzJ9ze5YEvH2WI6CHTAVcSu9ZCsXtgxu81CIvp6/k4eXsdfo7 +PvDCRD75yi41QitfzlcPp1OI7i/1/iQitqnr0iMgQ+A6wa+IKwwdxyk9IiXNAzgquTFU8NIxAVjM +osm1Zz526e+shQ4hKRVci69nPC3Kw4NQEmkQ65E7OodxorSvxjvpBjQHDmWFIQ1mlmzlS5vedseT +/mgIEsMJ7Lxz2bLAF9M5xeLEhdbHxpWOw0GdkJApMVBRF1y+a0z3c9WZPAXGFcFrJgCIB+024uad +0CrzmEoRa3Ub4swNIHPGf7QDV+2uj2OiFWsChgCwjKqN6rp5izpbH6Wc1O1TclQTP/XVwi6anTr1 +1sbubjZLI1+VptPSdCfwnFBrB1jvebrTA9uUhU2/9gad7xPqeFkaQcnnLbCViZK8d7R1kxzFrIJV +8EaLYmKYpvGVkig+3C5HCXbM1jGCGekiM2pRCVPyRyXYdPf6kcbWEQ36F5V4Gq9N7icNNw+JHwRE +LTgxRXACpvnQv/PuT0xCCAywY/K4hE6Now2qDwaSE5FB+1agsoUveYDepS83qFcF1NufvULD3fTl +g6Hgf7WBt6lzMeiyyWVn3P1WVbwaczHmTzE9A5SyItTVgFYyvs/L/fXlaNgbw8v3azT+0eikVlWD +/vBHbzQumP23uBCjsYdrL9OWARwxs/nuLOzeXbPJTa/Xv6sUmQir5pC1YRLz3eA+CD8Z0XpcW8v9 +MZWF36ryyXXf3yBIz6nzqz8Muyz0m5Qj7OexfYo/Ph3LqvkHUg7AuA== +""") + +MH_MAGIC = 0xfeedface +MH_CIGAM = 0xcefaedfe +MH_MAGIC_64 = 0xfeedfacf +MH_CIGAM_64 = 0xcffaedfe +FAT_MAGIC = 0xcafebabe +BIG_ENDIAN = '>' +LITTLE_ENDIAN = '<' +LC_LOAD_DYLIB = 0xc +maxint = majver == 3 and getattr(sys, 'maxsize') or getattr(sys, 'maxint') + + +class fileview(object): + """ + A proxy for file-like objects that exposes a given view of a file. + Modified from macholib. + """ + + def __init__(self, fileobj, start=0, size=maxint): + if isinstance(fileobj, fileview): + self._fileobj = fileobj._fileobj + else: + self._fileobj = fileobj + self._start = start + self._end = start + size + self._pos = 0 + + def __repr__(self): + return '<fileview [%d, %d] %r>' % ( + self._start, self._end, self._fileobj) + + def tell(self): + return self._pos + + def _checkwindow(self, seekto, op): + if not (self._start <= seekto <= self._end): + raise IOError("%s to offset %d is outside window [%d, %d]" % ( + op, seekto, self._start, self._end)) + + def seek(self, offset, whence=0): + seekto = offset + if whence == os.SEEK_SET: + seekto += self._start + elif whence == os.SEEK_CUR: + seekto += self._start + self._pos + elif whence == os.SEEK_END: + seekto += self._end + else: + raise IOError("Invalid whence argument to seek: %r" % (whence,)) + self._checkwindow(seekto, 'seek') + self._fileobj.seek(seekto) + self._pos = seekto - self._start + + def write(self, bytes): + here = self._start + self._pos + self._checkwindow(here, 'write') + self._checkwindow(here + len(bytes), 'write') + self._fileobj.seek(here, os.SEEK_SET) + self._fileobj.write(bytes) + self._pos += len(bytes) + + def read(self, size=maxint): + assert size >= 0 + here = self._start + self._pos + self._checkwindow(here, 'read') + size = min(size, self._end - here) + self._fileobj.seek(here, os.SEEK_SET) + bytes = self._fileobj.read(size) + self._pos += len(bytes) + return bytes + + +def read_data(file, endian, num=1): + """ + Read a given number of 32-bits unsigned integers from the given file + with the given endianness. + """ + res = struct.unpack(endian + 'L' * num, file.read(num * 4)) + if len(res) == 1: + return res[0] + return res + + +def mach_o_change(path, what, value): + """ + Replace a given name (what) in any LC_LOAD_DYLIB command found in + the given binary with a new name (value), provided it's shorter. + """ + + def do_macho(file, bits, endian): + # Read Mach-O header (the magic number is assumed read by the caller) + cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = read_data(file, endian, 6) + # 64-bits header has one more field. + if bits == 64: + read_data(file, endian) + # The header is followed by ncmds commands + for n in range(ncmds): + where = file.tell() + # Read command header + cmd, cmdsize = read_data(file, endian, 2) + if cmd == LC_LOAD_DYLIB: + # The first data field in LC_LOAD_DYLIB commands is the + # offset of the name, starting from the beginning of the + # command. + name_offset = read_data(file, endian) + file.seek(where + name_offset, os.SEEK_SET) + # Read the NUL terminated string + load = file.read(cmdsize - name_offset).decode() + load = load[:load.index('\0')] + # If the string is what is being replaced, overwrite it. + if load == what: + file.seek(where + name_offset, os.SEEK_SET) + file.write(value.encode() + '\0'.encode()) + # Seek to the next command + file.seek(where + cmdsize, os.SEEK_SET) + + def do_file(file, offset=0, size=maxint): + file = fileview(file, offset, size) + # Read magic number + magic = read_data(file, BIG_ENDIAN) + if magic == FAT_MAGIC: + # Fat binaries contain nfat_arch Mach-O binaries + nfat_arch = read_data(file, BIG_ENDIAN) + for n in range(nfat_arch): + # Read arch header + cputype, cpusubtype, offset, size, align = read_data(file, BIG_ENDIAN, 5) + do_file(file, offset, size) + elif magic == MH_MAGIC: + do_macho(file, 32, BIG_ENDIAN) + elif magic == MH_CIGAM: + do_macho(file, 32, LITTLE_ENDIAN) + elif magic == MH_MAGIC_64: + do_macho(file, 64, BIG_ENDIAN) + elif magic == MH_CIGAM_64: + do_macho(file, 64, LITTLE_ENDIAN) + + assert(len(what) >= len(value)) + + with open(path, 'r+b') as f: + do_file(f) + + +if __name__ == '__main__': + main() + +# TODO: +# Copy python.exe.manifest +# Monkeypatch distutils.sysconfig diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat new file mode 100644 index 000000000..529b9733c --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat @@ -0,0 +1,30 @@ +@echo off
+set "VIRTUAL_ENV=__VIRTUAL_ENV__"
+
+if defined _OLD_VIRTUAL_PROMPT (
+ set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
+) else (
+ if not defined PROMPT (
+ set "PROMPT=$P$G"
+ )
+ set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
+)
+set "PROMPT=__VIRTUAL_WINPROMPT__ %PROMPT%"
+
+REM Don't use () to avoid problems with them in %PATH%
+if defined _OLD_VIRTUAL_PYTHONHOME goto ENDIFVHOME
+ set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
+:ENDIFVHOME
+
+set PYTHONHOME=
+
+REM if defined _OLD_VIRTUAL_PATH (
+if not defined _OLD_VIRTUAL_PATH goto ENDIFVPATH1
+ set "PATH=%_OLD_VIRTUAL_PATH%"
+:ENDIFVPATH1
+REM ) else (
+if defined _OLD_VIRTUAL_PATH goto ENDIFVPATH2
+ set "_OLD_VIRTUAL_PATH=%PATH%"
+:ENDIFVPATH2
+
+set "PATH=%VIRTUAL_ENV%\__BIN_NAME__;%PATH%"
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh new file mode 100644 index 000000000..864865b17 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh @@ -0,0 +1,36 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi <davidedb@gmail.com>. + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "__VIRTUAL_ENV__" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/__BIN_NAME__:$PATH" + + + +if ("__VIRTUAL_PROMPT__" != "") then + set env_name = "__VIRTUAL_PROMPT__" +else + set env_name = `basename "$VIRTUAL_ENV"` +endif + +# Could be in a non-interactive environment, +# in which case, $prompt is undefined and we wouldn't +# care about the prompt anyway. +if ( $?prompt ) then + set _OLD_VIRTUAL_PROMPT="$prompt" + set prompt = "[$env_name] $prompt" +endif + +unset env_name + +alias pydoc python -m pydoc + +rehash + diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish new file mode 100644 index 000000000..f3d1797a3 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish @@ -0,0 +1,76 @@ +# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*. +# Do not run it directly. + +function deactivate -d 'Exit virtualenv mode and return to the normal environment.' + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`. + set -l fish_function_path + + # Erase virtualenv's `fish_prompt` and restore the original. + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + end + + set -e VIRTUAL_ENV + + if test "$argv[1]" != 'nondestructive' + # Self-destruct! + functions -e pydoc + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "__VIRTUAL_ENV__" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/__BIN_NAME__" $PATH + +# Unset `$PYTHONHOME` if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +function pydoc + python -m pydoc $argv +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # Copy the current `fish_prompt` function as `_old_fish_prompt`. + functions -c fish_prompt _old_fish_prompt + + function fish_prompt + # Save the current $status, for fish_prompts that display it. + set -l old_status $status + + # Prompt override provided? + # If not, just prepend the environment name. + if test -n "__VIRTUAL_PROMPT__" + printf '%s%s' "__VIRTUAL_PROMPT__" (set_color normal) + else + printf '%s(%s%s%s) ' (set_color normal) (set_color -o white) (basename "$VIRTUAL_ENV") (set_color normal) + end + + # Restore the original $status + echo "exit $old_status" | source + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1 b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1 new file mode 100644 index 000000000..0f4adf19f --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1 @@ -0,0 +1,150 @@ +# This file must be dot sourced from PoSh; you cannot run it
+# directly. Do this: . ./activate.ps1
+
+# FIXME: clean up unused vars.
+$script:THIS_PATH = $myinvocation.mycommand.path
+$script:BASE_DIR = split-path (resolve-path "$THIS_PATH/..") -Parent
+$script:DIR_NAME = split-path $BASE_DIR -Leaf
+
+function global:deactivate ( [switch] $NonDestructive ){
+
+ if ( test-path variable:_OLD_VIRTUAL_PATH ) {
+ $env:PATH = $variable:_OLD_VIRTUAL_PATH
+ remove-variable "_OLD_VIRTUAL_PATH" -scope global
+ }
+
+ if ( test-path function:_old_virtual_prompt ) {
+ $function:prompt = $function:_old_virtual_prompt
+ remove-item function:\_old_virtual_prompt
+ }
+
+ if ($env:VIRTUAL_ENV) {
+ $old_env = split-path $env:VIRTUAL_ENV -leaf
+ remove-item env:VIRTUAL_ENV -erroraction silentlycontinue
+ }
+
+ if ( !$NonDestructive ) {
+ # Self destruct!
+ remove-item function:deactivate
+ }
+}
+
+# unset irrelevant variables
+deactivate -nondestructive
+
+$VIRTUAL_ENV = $BASE_DIR
+$env:VIRTUAL_ENV = $VIRTUAL_ENV
+
+$global:_OLD_VIRTUAL_PATH = $env:PATH
+$env:PATH = "$env:VIRTUAL_ENV/Scripts;" + $env:PATH
+if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) {
+ function global:_old_virtual_prompt { "" }
+ $function:_old_virtual_prompt = $function:prompt
+ function global:prompt {
+ # Add a prefix to the current prompt, but don't discard it.
+ write-host "($(split-path $env:VIRTUAL_ENV -leaf)) " -nonewline
+ & $function:_old_virtual_prompt
+ }
+}
+
+# SIG # Begin signature block
+# MIISeAYJKoZIhvcNAQcCoIISaTCCEmUCAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB
+# gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR
+# AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUS5reBwSg3zOUwhXf2jPChZzf
+# yPmggg6tMIIGcDCCBFigAwIBAgIBJDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQG
+# EwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERp
+# Z2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2Vy
+# dGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjIwMTQ2WhcNMTcxMDI0MjIw
+# MTQ2WjCBjDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzAp
+# BgNVBAsTIlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNV
+# BAMTL1N0YXJ0Q29tIENsYXNzIDIgUHJpbWFyeSBJbnRlcm1lZGlhdGUgT2JqZWN0
+# IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyiOLIjUemqAbPJ1J
+# 0D8MlzgWKbr4fYlbRVjvhHDtfhFN6RQxq0PjTQxRgWzwFQNKJCdU5ftKoM5N4YSj
+# Id6ZNavcSa6/McVnhDAQm+8H3HWoD030NVOxbjgD/Ih3HaV3/z9159nnvyxQEckR
+# ZfpJB2Kfk6aHqW3JnSvRe+XVZSufDVCe/vtxGSEwKCaNrsLc9pboUoYIC3oyzWoU
+# TZ65+c0H4paR8c8eK/mC914mBo6N0dQ512/bkSdaeY9YaQpGtW/h/W/FkbQRT3sC
+# pttLVlIjnkuY4r9+zvqhToPjxcfDYEf+XD8VGkAqle8Aa8hQ+M1qGdQjAye8OzbV
+# uUOw7wIDAQABo4IB6TCCAeUwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC
+# AQYwHQYDVR0OBBYEFNBOD0CZbLhLGW87KLjg44gHNKq3MB8GA1UdIwQYMBaAFE4L
+# 7xqkQFulF2mHMMo0aEPQQa7yMD0GCCsGAQUFBwEBBDEwLzAtBggrBgEFBQcwAoYh
+# aHR0cDovL3d3dy5zdGFydHNzbC5jb20vc2ZzY2EuY3J0MFsGA1UdHwRUMFIwJ6Al
+# oCOGIWh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL3Nmc2NhLmNybDAnoCWgI4YhaHR0
+# cDovL2NybC5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMIGABgNVHSAEeTB3MHUGCysG
+# AQQBgbU3AQIBMGYwLgYIKwYBBQUHAgEWImh0dHA6Ly93d3cuc3RhcnRzc2wuY29t
+# L3BvbGljeS5wZGYwNAYIKwYBBQUHAgEWKGh0dHA6Ly93d3cuc3RhcnRzc2wuY29t
+# L2ludGVybWVkaWF0ZS5wZGYwEQYJYIZIAYb4QgEBBAQDAgABMFAGCWCGSAGG+EIB
+# DQRDFkFTdGFydENvbSBDbGFzcyAyIFByaW1hcnkgSW50ZXJtZWRpYXRlIE9iamVj
+# dCBTaWduaW5nIENlcnRpZmljYXRlczANBgkqhkiG9w0BAQUFAAOCAgEAcnMLA3Va
+# N4OIE9l4QT5OEtZy5PByBit3oHiqQpgVEQo7DHRsjXD5H/IyTivpMikaaeRxIv95
+# baRd4hoUcMwDj4JIjC3WA9FoNFV31SMljEZa66G8RQECdMSSufgfDYu1XQ+cUKxh
+# D3EtLGGcFGjjML7EQv2Iol741rEsycXwIXcryxeiMbU2TPi7X3elbwQMc4JFlJ4B
+# y9FhBzuZB1DV2sN2irGVbC3G/1+S2doPDjL1CaElwRa/T0qkq2vvPxUgryAoCppU
+# FKViw5yoGYC+z1GaesWWiP1eFKAL0wI7IgSvLzU3y1Vp7vsYaxOVBqZtebFTWRHt
+# XjCsFrrQBngt0d33QbQRI5mwgzEp7XJ9xu5d6RVWM4TPRUsd+DDZpBHm9mszvi9g
+# VFb2ZG7qRRXCSqys4+u/NLBPbXi/m/lU00cODQTlC/euwjk9HQtRrXQ/zqsBJS6U
+# J+eLGw1qOfj+HVBl/ZQpfoLk7IoWlRQvRL1s7oirEaqPZUIWY/grXq9r6jDKAp3L
+# ZdKQpPOnnogtqlU4f7/kLjEJhrrc98mrOWmVMK/BuFRAfQ5oDUMnVmCzAzLMjKfG
+# cVW/iMew41yfhgKbwpfzm3LBr1Zv+pEBgcgW6onRLSAn3XHM0eNtz+AkxH6rRf6B
+# 2mYhLEEGLapH8R1AMAo4BbVFOZR5kXcMCwowggg1MIIHHaADAgECAgIEuDANBgkq
+# hkiG9w0BAQUFADCBjDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0
+# ZC4xKzApBgNVBAsTIlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcx
+# ODA2BgNVBAMTL1N0YXJ0Q29tIENsYXNzIDIgUHJpbWFyeSBJbnRlcm1lZGlhdGUg
+# T2JqZWN0IENBMB4XDTExMTIwMzE1MzQxOVoXDTEzMTIwMzE0NTgwN1owgYwxIDAe
+# BgNVBA0TFzU4MTc5Ni1HaDd4Zkp4a3hRU0lPNEUwMQswCQYDVQQGEwJERTEPMA0G
+# A1UECBMGQmVybGluMQ8wDQYDVQQHEwZCZXJsaW4xFjAUBgNVBAMTDUphbm5pcyBM
+# ZWlkZWwxITAfBgkqhkiG9w0BCQEWEmphbm5pc0BsZWlkZWwuaW5mbzCCAiIwDQYJ
+# KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMcPeABYdN7nPq/AkZ/EkyUBGx/l2Yui
+# Lfm8ZdLG0ulMb/kQL3fRY7sUjYPyn9S6PhqqlFnNoGHJvbbReCdUC9SIQYmOEjEA
+# raHfb7MZU10NjO4U2DdGucj2zuO5tYxKizizOJF0e4yRQZVxpUGdvkW/+GLjCNK5
+# L7mIv3Z1dagxDKHYZT74HXiS4VFUwHF1k36CwfM2vsetdm46bdgSwV+BCMmZICYT
+# IJAS9UQHD7kP4rik3bFWjUx08NtYYFAVOd/HwBnemUmJe4j3IhZHr0k1+eDG8hDH
+# KVvPgLJIoEjC4iMFk5GWsg5z2ngk0LLu3JZMtckHsnnmBPHQK8a3opUNd8hdMNJx
+# gOwKjQt2JZSGUdIEFCKVDqj0FmdnDMPfwy+FNRtpBMl1sz78dUFhSrnM0D8NXrqa
+# 4rG+2FoOXlmm1rb6AFtpjAKksHRpYcPk2DPGWp/1sWB+dUQkS3gOmwFzyqeTuXpT
+# 0juqd3iAxOGx1VRFQ1VHLLf3AzV4wljBau26I+tu7iXxesVucSdsdQu293jwc2kN
+# xK2JyHCoZH+RyytrwS0qw8t7rMOukU9gwP8mn3X6mgWlVUODMcHTULjSiCEtvyZ/
+# aafcwjUbt4ReEcnmuZtWIha86MTCX7U7e+cnpWG4sIHPnvVTaz9rm8RyBkIxtFCB
+# nQ3FnoQgyxeJAgMBAAGjggOdMIIDmTAJBgNVHRMEAjAAMA4GA1UdDwEB/wQEAwIH
+# gDAuBgNVHSUBAf8EJDAiBggrBgEFBQcDAwYKKwYBBAGCNwIBFQYKKwYBBAGCNwoD
+# DTAdBgNVHQ4EFgQUWyCgrIWo8Ifvvm1/YTQIeMU9nc8wHwYDVR0jBBgwFoAU0E4P
+# QJlsuEsZbzsouODjiAc0qrcwggIhBgNVHSAEggIYMIICFDCCAhAGCysGAQQBgbU3
+# AQICMIIB/zAuBggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9s
+# aWN5LnBkZjA0BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50
+# ZXJtZWRpYXRlLnBkZjCB9wYIKwYBBQUHAgIwgeowJxYgU3RhcnRDb20gQ2VydGlm
+# aWNhdGlvbiBBdXRob3JpdHkwAwIBARqBvlRoaXMgY2VydGlmaWNhdGUgd2FzIGlz
+# c3VlZCBhY2NvcmRpbmcgdG8gdGhlIENsYXNzIDIgVmFsaWRhdGlvbiByZXF1aXJl
+# bWVudHMgb2YgdGhlIFN0YXJ0Q29tIENBIHBvbGljeSwgcmVsaWFuY2Ugb25seSBm
+# b3IgdGhlIGludGVuZGVkIHB1cnBvc2UgaW4gY29tcGxpYW5jZSBvZiB0aGUgcmVs
+# eWluZyBwYXJ0eSBvYmxpZ2F0aW9ucy4wgZwGCCsGAQUFBwICMIGPMCcWIFN0YXJ0
+# Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MAMCAQIaZExpYWJpbGl0eSBhbmQg
+# d2FycmFudGllcyBhcmUgbGltaXRlZCEgU2VlIHNlY3Rpb24gIkxlZ2FsIGFuZCBM
+# aW1pdGF0aW9ucyIgb2YgdGhlIFN0YXJ0Q29tIENBIHBvbGljeS4wNgYDVR0fBC8w
+# LTAroCmgJ4YlaHR0cDovL2NybC5zdGFydHNzbC5jb20vY3J0YzItY3JsLmNybDCB
+# iQYIKwYBBQUHAQEEfTB7MDcGCCsGAQUFBzABhitodHRwOi8vb2NzcC5zdGFydHNz
+# bC5jb20vc3ViL2NsYXNzMi9jb2RlL2NhMEAGCCsGAQUFBzAChjRodHRwOi8vYWlh
+# LnN0YXJ0c3NsLmNvbS9jZXJ0cy9zdWIuY2xhc3MyLmNvZGUuY2EuY3J0MCMGA1Ud
+# EgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzANBgkqhkiG9w0BAQUFAAOC
+# AQEAhrzEV6zwoEtKjnFRhCsjwiPykVpo5Eiye77Ve801rQDiRKgSCCiW6g3HqedL
+# OtaSs65Sj2pm3Viea4KR0TECLcbCTgsdaHqw2x1yXwWBQWZEaV6EB05lIwfr94P1
+# SFpV43zkuc+bbmA3+CRK45LOcCNH5Tqq7VGTCAK5iM7tvHwFlbQRl+I6VEL2mjpF
+# NsuRjDOVrv/9qw/a22YJ9R7Y1D0vUSs3IqZx2KMUaYDP7H2mSRxJO2nADQZBtriF
+# gTyfD3lYV12MlIi5CQwe3QC6DrrfSMP33i5Wa/OFJiQ27WPxmScYVhiqozpImFT4
+# PU9goiBv9RKXdgTmZE1PN0NQ5jGCAzUwggMxAgEBMIGTMIGMMQswCQYDVQQGEwJJ
+# TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0
+# YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg
+# MiBQcmltYXJ5IEludGVybWVkaWF0ZSBPYmplY3QgQ0ECAgS4MAkGBSsOAwIaBQCg
+# eDAYBgorBgEEAYI3AgEMMQowCKACgAChAoAAMBkGCSqGSIb3DQEJAzEMBgorBgEE
+# AYI3AgEEMBwGCisGAQQBgjcCAQsxDjAMBgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJ
+# BDEWBBRVGw0FDSiaIi38dWteRUAg/9Pr6DANBgkqhkiG9w0BAQEFAASCAgCInvOZ
+# FdaNFzbf6trmFDZKMojyx3UjKMCqNjHVBbuKY0qXwFC/ElYDV1ShJ2CBZbdurydO
+# OQ6cIQ0KREOCwmX/xB49IlLHHUxNhEkVv7HGU3EKAFf9IBt9Yr7jikiR9cjIsfHK
+# 4cjkoKJL7g28yEpLLkHt1eo37f1Ga9lDWEa5Zq3U5yX+IwXhrUBm1h8Xr033FhTR
+# VEpuSz6LHtbrL/zgJnCzJ2ahjtJoYevdcWiNXffosJHFaSfYDDbiNsPRDH/1avmb
+# 5j/7BhP8BcBaR6Fp8tFbNGIcWHHGcjqLMnTc4w13b7b4pDhypqElBa4+lCmwdvv9
+# GydYtRgPz8GHeoBoKj30YBlMzRIfFYaIFGIC4Ai3UEXkuH9TxYohVbGm/W0Kl4Lb
+# RJ1FwiVcLcTOJdgNId2vQvKc+jtNrjcg5SP9h2v/C4aTx8tyc6tE3TOPh2f9b8DL
+# S+SbVArJpuJqrPTxDDoO1QNjTgLcdVYeZDE+r/NjaGZ6cMSd8db3EaG3ijD/0bud
+# SItbm/OlNVbQOFRR76D+ZNgPcU5iNZ3bmvQQIg6aSB9MHUpIE/SeCkNl9YeVk1/1
+# GFULgNMRmIYP4KLvu9ylh5Gu3hvD5VNhH6+FlXANwFy07uXks5uF8mfZVxVCnodG
+# xkNCx+6PsrA5Z7WP4pXcmYnMn97npP/Q9EHJWw==
+# SIG # End signature block
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh new file mode 100644 index 000000000..477b7eca2 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh @@ -0,0 +1,78 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + unset -f pydoc >/dev/null 2>&1 + + # reset old environment variables + # ! [ -z ${VAR+_} ] returns true if VAR is declared at all + if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then + PATH="$_OLD_VIRTUAL_PATH" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then + PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then + hash -r 2>/dev/null + fi + + if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then + PS1="$_OLD_VIRTUAL_PS1" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="__VIRTUAL_ENV__" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/__BIN_NAME__:$PATH" +export PATH + +# unset PYTHONHOME if set +if ! [ -z "${PYTHONHOME+_}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then + _OLD_VIRTUAL_PS1="$PS1" + if [ "x__VIRTUAL_PROMPT__" != x ] ; then + PS1="__VIRTUAL_PROMPT__$PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1" + fi + export PS1 +fi + +# Make sure to unalias pydoc if it's already there +alias pydoc 2>/dev/null >/dev/null && unalias pydoc + +pydoc () { + python -m pydoc "$@" +} + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then + hash -r 2>/dev/null +fi diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py new file mode 100644 index 000000000..f18193bf8 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py @@ -0,0 +1,34 @@ +"""By using execfile(this_file, dict(__file__=this_file)) you will +activate this virtualenv environment. + +This can be used when you must use an existing Python interpreter, not +the virtualenv bin/python +""" + +try: + __file__ +except NameError: + raise AssertionError( + "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") +import sys +import os + +old_os_path = os.environ.get('PATH', '') +os.environ['PATH'] = os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path +base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') +else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') +prev_sys_path = list(sys.path) +import site +site.addsitedir(site_packages) +sys.real_prefix = sys.prefix +sys.prefix = base +# Move the added items to the front of the path: +new_sys_path = [] +for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) +sys.path[:0] = new_sys_path diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat new file mode 100644 index 000000000..9228d3171 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat @@ -0,0 +1,19 @@ +@echo off
+
+set VIRTUAL_ENV=
+
+REM Don't use () to avoid problems with them in %PATH%
+if not defined _OLD_VIRTUAL_PROMPT goto ENDIFVPROMPT
+ set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
+ set _OLD_VIRTUAL_PROMPT=
+:ENDIFVPROMPT
+
+if not defined _OLD_VIRTUAL_PYTHONHOME goto ENDIFVHOME
+ set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
+ set _OLD_VIRTUAL_PYTHONHOME=
+:ENDIFVHOME
+
+if not defined _OLD_VIRTUAL_PATH goto ENDIFVPATH
+ set "PATH=%_OLD_VIRTUAL_PATH%"
+ set _OLD_VIRTUAL_PATH=
+:ENDIFVPATH
\ No newline at end of file diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py new file mode 100644 index 000000000..29fc1da45 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py @@ -0,0 +1,101 @@ +import os +import sys +import warnings +import imp +import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib + # Important! To work on pypy, this must be a module that resides in the + # lib-python/modified-x.y.z directory + +dirname = os.path.dirname + +distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils') +if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)): + warnings.warn( + "The virtualenv distutils package at %s appears to be in the same location as the system distutils?") +else: + __path__.insert(0, distutils_path) + real_distutils = imp.load_module("_virtualenv_distutils", None, distutils_path, ('', '', imp.PKG_DIRECTORY)) + # Copy the relevant attributes + try: + __revision__ = real_distutils.__revision__ + except AttributeError: + pass + __version__ = real_distutils.__version__ + +from distutils import dist, sysconfig + +try: + basestring +except NameError: + basestring = str + +## patch build_ext (distutils doesn't know how to get the libs directory +## path on windows - it hardcodes the paths around the patched sys.prefix) + +if sys.platform == 'win32': + from distutils.command.build_ext import build_ext as old_build_ext + class build_ext(old_build_ext): + def finalize_options (self): + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, basestring): + self.library_dirs = self.library_dirs.split(os.pathsep) + + self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs")) + old_build_ext.finalize_options(self) + + from distutils.command import build_ext as build_ext_module + build_ext_module.build_ext = build_ext + +## distutils.dist patches: + +old_find_config_files = dist.Distribution.find_config_files +def find_config_files(self): + found = old_find_config_files(self) + system_distutils = os.path.join(distutils_path, 'distutils.cfg') + #if os.path.exists(system_distutils): + # found.insert(0, system_distutils) + # What to call the per-user config file + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + user_filename = os.path.join(sys.prefix, user_filename) + if os.path.isfile(user_filename): + for item in list(found): + if item.endswith('pydistutils.cfg'): + found.remove(item) + found.append(user_filename) + return found +dist.Distribution.find_config_files = find_config_files + +## distutils.sysconfig patches: + +old_get_python_inc = sysconfig.get_python_inc +def sysconfig_get_python_inc(plat_specific=0, prefix=None): + if prefix is None: + prefix = sys.real_prefix + return old_get_python_inc(plat_specific, prefix) +sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__ +sysconfig.get_python_inc = sysconfig_get_python_inc + +old_get_python_lib = sysconfig.get_python_lib +def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None): + if standard_lib and prefix is None: + prefix = sys.real_prefix + return old_get_python_lib(plat_specific, standard_lib, prefix) +sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__ +sysconfig.get_python_lib = sysconfig_get_python_lib + +old_get_config_vars = sysconfig.get_config_vars +def sysconfig_get_config_vars(*args): + real_vars = old_get_config_vars(*args) + if sys.platform == 'win32': + lib_dir = os.path.join(sys.real_prefix, "libs") + if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars: + real_vars['LIBDIR'] = lib_dir # asked for all + elif isinstance(real_vars, list) and 'LIBDIR' in args: + real_vars = real_vars + [lib_dir] # asked for list + return real_vars +sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__ +sysconfig.get_config_vars = sysconfig_get_config_vars diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg new file mode 100644 index 000000000..1af230ec9 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg @@ -0,0 +1,6 @@ +# This is a config file local to this virtualenv installation +# You may include options that will be used by all distutils commands, +# and by easy_install. For instance: +# +# [easy_install] +# find_links = http://mylocalsite diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config new file mode 100644 index 000000000..5e7a7c901 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config @@ -0,0 +1,78 @@ +#!__VIRTUAL_ENV__/__BIN_NAME__/python + +import sys +import getopt +import sysconfig + +valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', + 'ldflags', 'help'] + +if sys.version_info >= (3, 2): + valid_opts.insert(-1, 'extension-suffix') + valid_opts.append('abiflags') +if sys.version_info >= (3, 3): + valid_opts.append('configdir') + + +def exit_with_usage(code=1): + sys.stderr.write("Usage: {0} [{1}]\n".format( + sys.argv[0], '|'.join('--'+opt for opt in valid_opts))) + sys.exit(code) + +try: + opts, args = getopt.getopt(sys.argv[1:], '', valid_opts) +except getopt.error: + exit_with_usage() + +if not opts: + exit_with_usage() + +pyver = sysconfig.get_config_var('VERSION') +getvar = sysconfig.get_config_var + +opt_flags = [flag for (flag, val) in opts] + +if '--help' in opt_flags: + exit_with_usage(code=0) + +for opt in opt_flags: + if opt == '--prefix': + print(sysconfig.get_config_var('prefix')) + + elif opt == '--exec-prefix': + print(sysconfig.get_config_var('exec_prefix')) + + elif opt in ('--includes', '--cflags'): + flags = ['-I' + sysconfig.get_path('include'), + '-I' + sysconfig.get_path('platinclude')] + if opt == '--cflags': + flags.extend(getvar('CFLAGS').split()) + print(' '.join(flags)) + + elif opt in ('--libs', '--ldflags'): + abiflags = getattr(sys, 'abiflags', '') + libs = ['-lpython' + pyver + abiflags] + libs += getvar('LIBS').split() + libs += getvar('SYSLIBS').split() + # add the prefix/lib/pythonX.Y/config dir, but only if there is no + # shared library in prefix/lib/. + if opt == '--ldflags': + if not getvar('Py_ENABLE_SHARED'): + libs.insert(0, '-L' + getvar('LIBPL')) + if not getvar('PYTHONFRAMEWORK'): + libs.extend(getvar('LINKFORSHARED').split()) + print(' '.join(libs)) + + elif opt == '--extension-suffix': + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + if ext_suffix is None: + ext_suffix = sysconfig.get_config_var('SO') + print(ext_suffix) + + elif opt == '--abiflags': + if not getattr(sys, 'abiflags', None): + exit_with_usage() + print(sys.abiflags) + + elif opt == '--configdir': + print(sysconfig.get_config_var('LIBPL')) diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py new file mode 100644 index 000000000..7969769c3 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py @@ -0,0 +1,758 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +In earlier versions of Python (up to 1.5a3), scripts or modules that +needed to use site-specific modules would place ``import site'' +somewhere near the top of their code. Because of the automatic +import, this is no longer necessary (but code that does it still +works). + +This will append site-specific paths to the module search path. On +Unix, it starts with sys.prefix and sys.exec_prefix (if different) and +appends lib/python<version>/site-packages as well as lib/site-python. +It also supports the Debian convention of +lib/python<version>/dist-packages. On other platforms (mainly Mac and +Windows), it uses just sys.prefix (and sys.exec_prefix, if different, +but this is unlikely). The resulting directories, if they exist, are +appended to sys.path, and also inspected for path configuration files. + +FOR DEBIAN, this sys.path is augmented with directories in /usr/local. +Local addons go into /usr/local/lib/python<version>/site-packages +(resp. /usr/local/lib/site-python), Debian addons install into +/usr/{lib,share}/python<version>/dist-packages. + +A path configuration file is a file whose name has the form +<package>.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.X/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.X/site-packages/bar + /usr/local/lib/python2.X/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +After these path manipulations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. + +""" + +import sys +import os +try: + import __builtin__ as builtins +except ImportError: + import builtins +try: + set +except NameError: + from sets import Set as set + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = None +# for distutils.commands.install +USER_SITE = None +USER_BASE = None + +_is_64bit = (getattr(sys, 'maxsize', None) or getattr(sys, 'maxint')) > 2**32 +_is_pypy = hasattr(sys, 'pypy_version_info') +_is_jython = sys.platform[:4] == 'java' +if _is_jython: + ModuleType = type(os) + +def makepath(*paths): + dir = os.path.join(*paths) + if _is_jython and (dir == '__classpath__' or + dir.startswith('__pyclasspath__')): + return dir, dir + dir = os.path.abspath(dir) + return dir, os.path.normcase(dir) + +def abs__file__(): + """Set all module' __file__ attribute to an absolute path""" + for m in sys.modules.values(): + if ((_is_jython and not isinstance(m, ModuleType)) or + hasattr(m, '__loader__')): + # only modules need the abspath in Jython. and don't mess + # with a PEP 302-supplied __file__ + continue + f = getattr(m, '__file__', None) + if f is None: + continue + m.__file__ = os.path.abspath(f) + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if not dircase in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + +# XXX This should not be part of site.py, since it is needed even when +# using the -S option for Python. See http://www.python.org/sf/586680 +def addbuilddir(): + """Append ./build/lib.<platform> in case we're running in the build dir + (especially for Guido :-)""" + from distutils.util import get_platform + s = "build/lib.%s-%.3s" % (get_platform(), sys.version) + if hasattr(sys, 'gettotalrefcount'): + s += '-pydebug' + s = os.path.join(os.path.dirname(sys.path[-1]), s) + sys.path.append(s) + +def _init_pathinfo(): + """Return a set containing all existing directory entries from sys.path""" + d = set() + for dir in sys.path: + try: + if os.path.isdir(dir): + dir, dircase = makepath(dir) + d.add(dircase) + except TypeError: + continue + return d + +def addpackage(sitedir, name, known_paths): + """Add a new path to known_paths by combining sitedir and 'name' or execute + sitedir if it starts with 'import'""" + if known_paths is None: + _init_pathinfo() + reset = 1 + else: + reset = 0 + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "rU") + except IOError: + return + try: + for line in f: + if line.startswith("#"): + continue + if line.startswith("import"): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + finally: + f.close() + if reset: + known_paths = None + return known_paths + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = 1 + else: + reset = 0 + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + try: + names = os.listdir(sitedir) + except os.error: + return + names.sort() + for name in names: + if name.endswith(os.extsep + "pth"): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + +def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix): + """Add site-packages (and possibly site-python) to sys.path""" + prefixes = [os.path.join(sys_prefix, "local"), sys_prefix] + if exec_prefix != sys_prefix: + prefixes.append(os.path.join(exec_prefix, "local")) + + for prefix in prefixes: + if prefix: + if sys.platform in ('os2emx', 'riscos') or _is_jython: + sitedirs = [os.path.join(prefix, "Lib", "site-packages")] + elif _is_pypy: + sitedirs = [os.path.join(prefix, 'site-packages')] + elif sys.platform == 'darwin' and prefix == sys_prefix: + + if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python + + sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"), + os.path.join(prefix, "Extras", "lib", "python")] + + else: # any other Python distros on OSX work this way + sitedirs = [os.path.join(prefix, "lib", + "python" + sys.version[:3], "site-packages")] + + elif os.sep == '/': + sitedirs = [os.path.join(prefix, + "lib", + "python" + sys.version[:3], + "site-packages"), + os.path.join(prefix, "lib", "site-python"), + os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")] + lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages") + if (os.path.exists(lib64_dir) and + os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]): + if _is_64bit: + sitedirs.insert(0, lib64_dir) + else: + sitedirs.append(lib64_dir) + try: + # sys.getobjects only available in --with-pydebug build + sys.getobjects + sitedirs.insert(0, os.path.join(sitedirs[0], 'debug')) + except AttributeError: + pass + # Debian-specific dist-packages directories: + sitedirs.append(os.path.join(prefix, "local/lib", + "python" + sys.version[:3], + "dist-packages")) + if sys.version[0] == '2': + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[:3], + "dist-packages")) + else: + sitedirs.append(os.path.join(prefix, "lib", + "python" + sys.version[0], + "dist-packages")) + sitedirs.append(os.path.join(prefix, "lib", "dist-python")) + else: + sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")] + if sys.platform == 'darwin': + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' in prefix: + home = os.environ.get('HOME') + if home: + sitedirs.append( + os.path.join(home, + 'Library', + 'Python', + sys.version[:3], + 'site-packages')) + for sitedir in sitedirs: + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + return None + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False): + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + + USER_BASE is the root directory for all Python versions + + USER_SITE is the user specific site-packages directory + + USER_SITE/.. can be used for data. + """ + global USER_BASE, USER_SITE, ENABLE_USER_SITE + env_base = os.environ.get("PYTHONUSERBASE", None) + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + #if sys.platform in ('os2emx', 'riscos'): + # # Don't know what to put here + # USER_BASE = '' + # USER_SITE = '' + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser(base, "Python") + USER_SITE = os.path.join(USER_BASE, + "Python" + sys.version[0] + sys.version[2], + "site-packages") + else: + if env_base: + USER_BASE = env_base + else: + USER_BASE = joinuser("~", ".local") + USER_SITE = os.path.join(USER_BASE, "lib", + "python" + sys.version[:3], + "site-packages") + + if ENABLE_USER_SITE and os.path.isdir(USER_SITE): + addsitedir(USER_SITE, known_paths) + if ENABLE_USER_SITE: + for dist_libdir in ("lib", "local/lib"): + user_site = os.path.join(USER_BASE, dist_libdir, + "python" + sys.version[:3], + "dist-packages") + if os.path.isdir(user_site): + addsitedir(user_site, known_paths) + return known_paths + + + +def setBEGINLIBPATH(): + """The OS/2 EMX port has optional extension modules that do double duty + as DLLs (and must use the .DLL file extension) for other extensions. + The library search path needs to be amended so these will be found + during module import. Use BEGINLIBPATH so that these are at the start + of the library search path. + + """ + dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload") + libpath = os.environ['BEGINLIBPATH'].split(';') + if libpath[-1]: + libpath.append(dllpath) + else: + libpath[-1] = dllpath + os.environ['BEGINLIBPATH'] = ';'.join(libpath) + + +def setquit(): + """Define new built-ins 'quit' and 'exit'. + These are simply strings that display a hint on how to exit. + + """ + if os.sep == ':': + eof = 'Cmd-Q' + elif os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + class Quitter(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return 'Use %s() or %s to exit' % (self.name, eof) + def __call__(self, code=None): + # Shells like IDLE catch the SystemExit, but listen when their + # stdin wrapper is closed. + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + builtins.quit = Quitter('quit') + builtins.exit = Quitter('exit') + + +class _Printer(object): + """interactive prompt objects for printing the license text, a list of + contributors and the copyright notice.""" + + MAXLINES = 23 + + def __init__(self, name, data, files=(), dirs=()): + self.__name = name + self.__data = data + self.__files = files + self.__dirs = dirs + self.__lines = None + + def __setup(self): + if self.__lines: + return + data = None + for dir in self.__dirs: + for filename in self.__files: + filename = os.path.join(dir, filename) + try: + fp = open(filename, "rU") + data = fp.read() + fp.close() + break + except IOError: + pass + if data: + break + if not data: + data = self.__data + self.__lines = data.split('\n') + self.__linecnt = len(self.__lines) + + def __repr__(self): + self.__setup() + if len(self.__lines) <= self.MAXLINES: + return "\n".join(self.__lines) + else: + return "Type %s() to see the full %s text" % ((self.__name,)*2) + + def __call__(self): + self.__setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print(self.__lines[i]) + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + try: + key = raw_input(prompt) + except NameError: + key = input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break + +def setcopyright(): + """Set 'copyright' and 'credits' in __builtin__""" + builtins.copyright = _Printer("copyright", sys.copyright) + if _is_jython: + builtins.credits = _Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + elif _is_pypy: + builtins.credits = _Printer( + "credits", + "PyPy is maintained by the PyPy developers: http://pypy.org/") + else: + builtins.credits = _Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + here = os.path.dirname(os.__file__) + builtins.license = _Printer( + "license", "See http://www.python.org/%.3s/license.html" % sys.version, + ["LICENSE.txt", "LICENSE"], + [os.path.join(here, os.pardir), here, os.curdir]) + + +class _Helper(object): + """Define the built-in 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + +def sethelper(): + builtins.help = _Helper() + +def aliasmbcs(): + """On Windows, some default encodings are not provided by Python, + while they are always available as "mbcs" in each locale. Make + them usable by aliasing to "mbcs" in such a case.""" + if sys.platform == 'win32': + import locale, codecs + enc = locale.getdefaultlocale()[1] + if enc.startswith('cp'): # "cp***" ? + try: + codecs.lookup(enc) + except LookupError: + import encodings + encodings._cache[enc] = encodings._unknown + encodings.aliases.aliases[enc] = 'mbcs' + +def setencoding(): + """Set the string encoding used by the Unicode implementation. The + default is 'ascii', but if you're willing to experiment, you can + change this.""" + encoding = "ascii" # Default value set by _PyUnicode_Init() + if 0: + # Enable to support locale aware default string encodings. + import locale + loc = locale.getdefaultlocale() + if loc[1]: + encoding = loc[1] + if 0: + # Enable to switch off string to Unicode coercion and implicit + # Unicode to string conversion. + encoding = "undefined" + if encoding != "ascii": + # On Non-Unicode builds this will raise an AttributeError... + sys.setdefaultencoding(encoding) # Needs Python Unicode build ! + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + import sitecustomize + except ImportError: + pass + +def virtual_install_main_packages(): + f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt')) + sys.real_prefix = f.read().strip() + f.close() + pos = 2 + hardcoded_relative_dirs = [] + if sys.path[0] == '': + pos += 1 + if _is_jython: + paths = [os.path.join(sys.real_prefix, 'Lib')] + elif _is_pypy: + if sys.version_info > (3, 2): + cpyver = '%d' % sys.version_info[0] + elif sys.pypy_version_info >= (1, 5): + cpyver = '%d.%d' % sys.version_info[:2] + else: + cpyver = '%d.%d.%d' % sys.version_info[:3] + paths = [os.path.join(sys.real_prefix, 'lib_pypy'), + os.path.join(sys.real_prefix, 'lib-python', cpyver)] + if sys.pypy_version_info < (1, 9): + paths.insert(1, os.path.join(sys.real_prefix, + 'lib-python', 'modified-%s' % cpyver)) + hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below + # + # This is hardcoded in the Python executable, but relative to sys.prefix: + for path in paths[:]: + plat_path = os.path.join(path, 'plat-%s' % sys.platform) + if os.path.exists(plat_path): + paths.append(plat_path) + elif sys.platform == 'win32': + paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')] + else: + paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])] + hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below + lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3]) + if os.path.exists(lib64_path): + if _is_64bit: + paths.insert(0, lib64_path) + else: + paths.append(lib64_path) + # This is hardcoded in the Python executable, but relative to + # sys.prefix. Debian change: we need to add the multiarch triplet + # here, which is where the real stuff lives. As per PEP 421, in + # Python 3.3+, this lives in sys.implementation, while in Python 2.7 + # it lives in sys. + try: + arch = getattr(sys, 'implementation', sys)._multiarch + except AttributeError: + # This is a non-multiarch aware Python. Fallback to the old way. + arch = sys.platform + plat_path = os.path.join(sys.real_prefix, 'lib', + 'python'+sys.version[:3], + 'plat-%s' % arch) + if os.path.exists(plat_path): + paths.append(plat_path) + # This is hardcoded in the Python executable, but + # relative to sys.prefix, so we have to fix up: + for path in list(paths): + tk_dir = os.path.join(path, 'lib-tk') + if os.path.exists(tk_dir): + paths.append(tk_dir) + + # These are hardcoded in the Apple's Python executable, + # but relative to sys.prefix, so we have to fix them up: + if sys.platform == 'darwin': + hardcoded_paths = [os.path.join(relative_dir, module) + for relative_dir in hardcoded_relative_dirs + for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')] + + for path in hardcoded_paths: + if os.path.exists(path): + paths.append(path) + + sys.path.extend(paths) + +def force_global_eggs_after_local_site_packages(): + """ + Force easy_installed eggs in the global environment to get placed + in sys.path after all packages inside the virtualenv. This + maintains the "least surprise" result that packages in the + virtualenv always mask global packages, never the other way + around. + + """ + egginsert = getattr(sys, '__egginsert', 0) + for i, path in enumerate(sys.path): + if i > egginsert and path.startswith(sys.prefix): + egginsert = i + sys.__egginsert = egginsert + 1 + +def virtual_addsitepackages(known_paths): + force_global_eggs_after_local_site_packages() + return addsitepackages(known_paths, sys_prefix=sys.real_prefix) + +def fixclasspath(): + """Adjust the special classpath sys.path entries for Jython. These + entries should follow the base virtualenv lib directories. + """ + paths = [] + classpaths = [] + for path in sys.path: + if path == '__classpath__' or path.startswith('__pyclasspath__'): + classpaths.append(path) + else: + paths.append(path) + sys.path = paths + sys.path.extend(classpaths) + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + import usercustomize + except ImportError: + pass + + +def main(): + global ENABLE_USER_SITE + virtual_install_main_packages() + abs__file__() + paths_in_sys = removeduppaths() + if (os.name == "posix" and sys.path and + os.path.basename(sys.path[-1]) == "Modules"): + addbuilddir() + if _is_jython: + fixclasspath() + GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt')) + if not GLOBAL_SITE_PACKAGES: + ENABLE_USER_SITE = False + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + paths_in_sys = addsitepackages(paths_in_sys) + paths_in_sys = addusersitepackages(paths_in_sys) + if GLOBAL_SITE_PACKAGES: + paths_in_sys = virtual_addsitepackages(paths_in_sys) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding + +main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + print("sys.path = [") + for dir in sys.path: + print(" %r," % (dir,)) + print("]") + def exists(path): + if os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + print("USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE))) + print("USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE))) + print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE) + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print(os.pathsep.join(buffer)) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py b/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl Binary files differnew file mode 100644 index 000000000..dfef51d44 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl Binary files differnew file mode 100644 index 000000000..cc49227a0 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl Binary files differnew file mode 100644 index 000000000..02c8ce873 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl Binary files differnew file mode 100644 index 000000000..506d5e520 --- /dev/null +++ b/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl |