summaryrefslogtreecommitdiffstats
path: root/build/subconfigure.py
diff options
context:
space:
mode:
Diffstat (limited to 'build/subconfigure.py')
-rw-r--r--build/subconfigure.py441
1 files changed, 441 insertions, 0 deletions
diff --git a/build/subconfigure.py b/build/subconfigure.py
new file mode 100644
index 000000000..6c152f406
--- /dev/null
+++ b/build/subconfigure.py
@@ -0,0 +1,441 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to capture the content of config.status-generated
+# files and subsequently restore their timestamp if they haven't changed.
+
+import argparse
+import errno
+import itertools
+import os
+import re
+import subprocess
+import sys
+import pickle
+
+import mozpack.path as mozpath
+
+
+class Pool(object):
+ def __new__(cls, size):
+ try:
+ import multiprocessing
+ size = min(size, multiprocessing.cpu_count())
+ return multiprocessing.Pool(size)
+ except:
+ return super(Pool, cls).__new__(cls)
+
+ def imap_unordered(self, fn, iterable):
+ return itertools.imap(fn, iterable)
+
+ def close(self):
+ pass
+
+ def join(self):
+ pass
+
+
+class File(object):
+ def __init__(self, path):
+ self._path = path
+ self._content = open(path, 'rb').read()
+ stat = os.stat(path)
+ self._times = (stat.st_atime, stat.st_mtime)
+
+ @property
+ def path(self):
+ return self._path
+
+ @property
+ def mtime(self):
+ return self._times[1]
+
+ @property
+ def modified(self):
+ '''Returns whether the file was modified since the instance was
+ created. Result is memoized.'''
+ if hasattr(self, '_modified'):
+ return self._modified
+
+ modified = True
+ if os.path.exists(self._path):
+ if open(self._path, 'rb').read() == self._content:
+ modified = False
+ self._modified = modified
+ return modified
+
+ def update_time(self):
+ '''If the file hasn't changed since the instance was created,
+ restore its old modification time.'''
+ if not self.modified:
+ os.utime(self._path, self._times)
+
+
+# As defined in the various sub-configures in the tree
+PRECIOUS_VARS = set([
+ 'build_alias',
+ 'host_alias',
+ 'target_alias',
+ 'CC',
+ 'CFLAGS',
+ 'LDFLAGS',
+ 'LIBS',
+ 'CPPFLAGS',
+ 'CPP',
+ 'CCC',
+ 'CXXFLAGS',
+ 'CXX',
+ 'CCASFLAGS',
+ 'CCAS',
+])
+
+
+CONFIGURE_DATA = 'configure.pkl'
+
+
+# Autoconf, in some of the sub-configures used in the tree, likes to error
+# out when "precious" variables change in value. The solution it gives to
+# straighten things is to either run make distclean or remove config.cache.
+# There's no reason not to do the latter automatically instead of failing,
+# doing the cleanup (which, on buildbots means a full clobber), and
+# restarting from scratch.
+def maybe_clear_cache(data):
+ env = dict(data['env'])
+ for kind in ('target', 'host', 'build'):
+ arg = data[kind]
+ if arg is not None:
+ env['%s_alias' % kind] = arg
+ # configure can take variables assignments in its arguments, and that
+ # overrides whatever is in the environment.
+ for arg in data['args']:
+ if arg[:1] != '-' and '=' in arg:
+ key, value = arg.split('=', 1)
+ env[key] = value
+
+ comment = re.compile(r'^\s+#')
+ cache = {}
+ with open(data['cache-file']) as f:
+ for line in f:
+ if not comment.match(line) and '=' in line:
+ key, value = line.rstrip(os.linesep).split('=', 1)
+ # If the value is quoted, unquote it
+ if value[:1] == "'":
+ value = value[1:-1].replace("'\\''", "'")
+ cache[key] = value
+ for precious in PRECIOUS_VARS:
+ # If there is no entry at all for that precious variable, then
+ # its value is not precious for that particular configure.
+ if 'ac_cv_env_%s_set' % precious not in cache:
+ continue
+ is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
+ value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
+ if value != env.get(precious):
+ print 'Removing %s because of %s value change from:' \
+ % (data['cache-file'], precious)
+ print ' %s' % (value if value is not None else 'undefined')
+ print 'to:'
+ print ' %s' % env.get(precious, 'undefined')
+ os.remove(data['cache-file'])
+ return True
+ return False
+
+
+def split_template(s):
+ """Given a "file:template" string, returns "file", "template". If the string
+ is of the form "file" (without a template), returns "file", "file.in"."""
+ if ':' in s:
+ return s.split(':', 1)
+ return s, '%s.in' % s
+
+
+def get_config_files(data):
+ config_status = mozpath.join(data['objdir'], 'config.status')
+ if not os.path.exists(config_status):
+ return [], []
+
+ configure = mozpath.join(data['srcdir'], 'configure')
+ config_files = []
+ command_files = []
+
+ # Scan the config.status output for information about configuration files
+ # it generates.
+ config_status_output = subprocess.check_output(
+ [data['shell'], '-c', '%s --help' % config_status],
+ stderr=subprocess.STDOUT).splitlines()
+ state = None
+ for line in config_status_output:
+ if line.startswith('Configuration') and line.endswith(':'):
+ if line.endswith('commands:'):
+ state = 'commands'
+ else:
+ state = 'config'
+ elif not line.strip():
+ state = None
+ elif state:
+ for f, t in (split_template(couple) for couple in line.split()):
+ f = mozpath.join(data['objdir'], f)
+ t = mozpath.join(data['srcdir'], t)
+ if state == 'commands':
+ command_files.append(f)
+ else:
+ config_files.append((f, t))
+
+ return config_files, command_files
+
+
+def prepare(srcdir, objdir, shell, args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--target', type=str)
+ parser.add_argument('--host', type=str)
+ parser.add_argument('--build', type=str)
+ parser.add_argument('--cache-file', type=str)
+ # The --srcdir argument is simply ignored. It's a useless autoconf feature
+ # that we don't support well anyways. This makes it stripped from `others`
+ # and allows to skip setting it when calling the subconfigure (configure
+ # will take it from the configure path anyways).
+ parser.add_argument('--srcdir', type=str)
+
+ data_file = os.path.join(objdir, CONFIGURE_DATA)
+ previous_args = None
+ if os.path.exists(data_file):
+ with open(data_file, 'rb') as f:
+ data = pickle.load(f)
+ previous_args = data['args']
+
+ # Msys likes to break environment variables and command line arguments,
+ # so read those from stdin, as they are passed from the configure script
+ # when necessary (on windows).
+ input = sys.stdin.read()
+ if input:
+ data = {a: b for [a, b] in eval(input)}
+ environ = {a: b for a, b in data['env']}
+ # These environment variables as passed from old-configure may contain
+ # posix-style paths, which will not be meaningful to the js
+ # subconfigure, which runs as a native python process, so use their
+ # values from the environment. In the case of autoconf implemented
+ # subconfigures, Msys will re-convert them properly.
+ for var in ('HOME', 'TERM', 'PATH', 'TMPDIR', 'TMP',
+ 'TEMP', 'INCLUDE'):
+ if var in environ and var in os.environ:
+ environ[var] = os.environ[var]
+ args = data['args']
+ else:
+ environ = os.environ
+
+ args, others = parser.parse_known_args(args)
+
+ data = {
+ 'target': args.target,
+ 'host': args.host,
+ 'build': args.build,
+ 'args': others,
+ 'shell': shell,
+ 'srcdir': srcdir,
+ 'env': environ,
+ }
+
+ if args.cache_file:
+ data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
+ args.cache_file))
+ else:
+ data['cache-file'] = mozpath.join(objdir, 'config.cache')
+
+ if previous_args is not None:
+ data['previous-args'] = previous_args
+
+ try:
+ os.makedirs(objdir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ with open(data_file, 'wb') as f:
+ pickle.dump(data, f)
+
+
+def prefix_lines(text, prefix):
+ return ''.join('%s> %s' % (prefix, line) for line in text.splitlines(True))
+
+
+def run(objdir):
+ ret = 0
+ output = ''
+
+ with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
+ data = pickle.load(f)
+
+ data['objdir'] = objdir
+
+ cache_file = data['cache-file']
+ cleared_cache = True
+ if os.path.exists(cache_file):
+ cleared_cache = maybe_clear_cache(data)
+
+ config_files, command_files = get_config_files(data)
+ contents = []
+ for f, t in config_files:
+ contents.append(File(f))
+
+ # AC_CONFIG_COMMANDS actually only registers tags, not file names
+ # but most commands are tagged with the file name they create.
+ # However, a few don't, or are tagged with a directory name (and their
+ # command is just to create that directory)
+ for f in command_files:
+ if os.path.isfile(f):
+ contents.append(File(f))
+
+ # Only run configure if one of the following is true:
+ # - config.status doesn't exist
+ # - config.status is older than configure
+ # - the configure arguments changed
+ # - the environment changed in a way that requires a cache clear.
+ configure = mozpath.join(data['srcdir'], 'configure')
+ config_status_path = mozpath.join(objdir, 'config.status')
+ skip_configure = True
+ if not os.path.exists(config_status_path):
+ skip_configure = False
+ config_status = None
+ else:
+ config_status = File(config_status_path)
+ if config_status.mtime < os.path.getmtime(configure) or \
+ data.get('previous-args', data['args']) != data['args'] or \
+ cleared_cache:
+ skip_configure = False
+
+ relobjdir = os.path.relpath(objdir, os.getcwd())
+
+ if not skip_configure:
+ if mozpath.normsep(relobjdir) == 'js/src':
+ # Because configure is a shell script calling a python script
+ # calling a shell script, on Windows, with msys screwing the
+ # environment, we lose the benefits from our own efforts in this
+ # script to get past the msys problems. So manually call the python
+ # script instead, so that we don't do a native->msys transition
+ # here. Then the python configure will still have the right
+ # environment when calling the shell configure.
+ command = [
+ sys.executable,
+ os.path.join(os.path.dirname(__file__), '..', 'configure.py'),
+ '--enable-project=js',
+ ]
+ data['env']['OLD_CONFIGURE'] = os.path.join(
+ os.path.dirname(configure), 'old-configure')
+ else:
+ command = [data['shell'], configure]
+ for kind in ('target', 'build', 'host'):
+ if data.get(kind) is not None:
+ command += ['--%s=%s' % (kind, data[kind])]
+ command += data['args']
+ command += ['--cache-file=%s' % cache_file]
+
+ # Pass --no-create to configure so that it doesn't run config.status.
+ # We're going to run it ourselves.
+ command += ['--no-create']
+
+ print prefix_lines('configuring', relobjdir)
+ print prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir)
+ sys.stdout.flush()
+ try:
+ output += subprocess.check_output(command,
+ stderr=subprocess.STDOUT, cwd=objdir, env=data['env'])
+ except subprocess.CalledProcessError as e:
+ return relobjdir, e.returncode, e.output
+
+ # Leave config.status with a new timestamp if configure is newer than
+ # its original mtime.
+ if config_status and os.path.getmtime(configure) <= config_status.mtime:
+ config_status.update_time()
+
+ # Only run config.status if one of the following is true:
+ # - config.status changed or did not exist
+ # - one of the templates for config files is newer than the corresponding
+ # config file.
+ skip_config_status = True
+ if not config_status or config_status.modified:
+ # If config.status doesn't exist after configure (because it's not
+ # an autoconf configure), skip it.
+ if os.path.exists(config_status_path):
+ skip_config_status = False
+ else:
+ # config.status changed or was created, so we need to update the
+ # list of config and command files.
+ config_files, command_files = get_config_files(data)
+ for f, t in config_files:
+ if not os.path.exists(t) or \
+ os.path.getmtime(f) < os.path.getmtime(t):
+ skip_config_status = False
+
+ if not skip_config_status:
+ if skip_configure:
+ print prefix_lines('running config.status', relobjdir)
+ sys.stdout.flush()
+ try:
+ output += subprocess.check_output([data['shell'], '-c',
+ './config.status'], stderr=subprocess.STDOUT, cwd=objdir,
+ env=data['env'])
+ except subprocess.CalledProcessError as e:
+ ret = e.returncode
+ output += e.output
+
+ for f in contents:
+ f.update_time()
+
+ return relobjdir, ret, output
+
+
+def subconfigure(args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--list', type=str,
+ help='File containing a list of subconfigures to run')
+ parser.add_argument('--skip', type=str,
+ help='File containing a list of Subconfigures to skip')
+ parser.add_argument('subconfigures', type=str, nargs='*',
+ help='Subconfigures to run if no list file is given')
+ args, others = parser.parse_known_args(args)
+ subconfigures = args.subconfigures
+ if args.list:
+ subconfigures.extend(open(args.list, 'rb').read().splitlines())
+ if args.skip:
+ skips = set(open(args.skip, 'rb').read().splitlines())
+ subconfigures = [s for s in subconfigures if s not in skips]
+
+ if not subconfigures:
+ return 0
+
+ ret = 0
+ # One would think using a ThreadPool would be faster, considering
+ # everything happens in subprocesses anyways, but no, it's actually
+ # slower on Windows. (20s difference overall!)
+ pool = Pool(len(subconfigures))
+ for relobjdir, returncode, output in \
+ pool.imap_unordered(run, subconfigures):
+ print prefix_lines(output, relobjdir)
+ sys.stdout.flush()
+ ret = max(returncode, ret)
+ if ret:
+ break
+ pool.close()
+ pool.join()
+ return ret
+
+
+def main(args):
+ if args[0] != '--prepare':
+ return subconfigure(args)
+
+ topsrcdir = os.path.abspath(args[1])
+ subdir = args[2]
+ # subdir can be of the form srcdir:objdir
+ if ':' in subdir:
+ srcdir, subdir = subdir.split(':', 1)
+ else:
+ srcdir = subdir
+ srcdir = os.path.join(topsrcdir, srcdir)
+ objdir = os.path.abspath(subdir)
+
+ return prepare(srcdir, objdir, args[3], args[4:])
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))