diff options
Diffstat (limited to 'build/util')
| -rw-r--r-- | build/util/count_ctors.py | 64 | ||||
| -rw-r--r-- | build/util/hg.py | 611 |
2 files changed, 675 insertions, 0 deletions
diff --git a/build/util/count_ctors.py b/build/util/count_ctors.py new file mode 100644 index 000000000..6a3a87068 --- /dev/null +++ b/build/util/count_ctors.py @@ -0,0 +1,64 @@ + +#!/usr/bin/python +import json +import re +import subprocess +import sys + +def count_ctors(filename): + proc = subprocess.Popen( + ['readelf', '-W', '-S', filename], stdout=subprocess.PIPE) + + # Some versions of ld produce both .init_array and .ctors. So we have + # to check for both. + n_init_array_ctors = 0 + have_init_array = False + n_ctors_ctors = 0 + have_ctors = False + + for line in proc.stdout: + f = line.split() + if len(f) != 11: + continue + # Don't try to int()-parse the header line for the section summaries. + if not re.match("\\[\\d+\\]", f[0]): + continue + section_name, contents, size, align = f[1], f[2], int(f[5], 16), int(f[10]) + if section_name == ".ctors" and contents == "PROGBITS": + have_ctors = True + # Subtract 2 for the uintptr_t(-1) header and the null terminator. + n_ctors_ctors = size / align - 2 + if section_name == ".init_array" and contents == "INIT_ARRAY": + have_init_array = True + n_init_array_ctors = size / align + + if have_init_array: + # Even if we have .ctors, we shouldn't have any constructors in .ctors. + # Complain if .ctors does not look how we expect it to. + if have_ctors and n_ctors_ctors != 0: + print >>sys.stderr, "Unexpected .ctors contents for", filename + sys.exit(1) + return n_init_array_ctors + if have_ctors: + return n_ctors_ctors + + # We didn't find anything; somebody switched initialization mechanisms on + # us, or the binary is completely busted. Complain either way. + print >>sys.stderr, "Couldn't find .init_array or .ctors in", filename + sys.exit(1) + +if __name__ == '__main__': + for f in sys.argv[1:]: + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [{ + "name": "compiler_metrics", + "subtests": [{ + "name": "num_constructors", + "value": count_ctors(f), + "alertThreshold": 0.25 + }]} + ] + } + print "PERFHERDER_DATA: %s" % json.dumps(perfherder_data) + diff --git a/build/util/hg.py b/build/util/hg.py new file mode 100644 index 000000000..c76c8ef03 --- /dev/null +++ b/build/util/hg.py @@ -0,0 +1,611 @@ +"""Functions for interacting with hg""" +import os +import re +import subprocess +from urlparse import urlsplit +from ConfigParser import RawConfigParser + +from util.commands import run_cmd, get_output, remove_path +from util.retry import retry + +import logging +log = logging.getLogger(__name__) + + +class DefaultShareBase: + pass +DefaultShareBase = DefaultShareBase() + + +class HgUtilError(Exception): + pass + + +def _make_absolute(repo): + if repo.startswith("file://"): + path = repo[len("file://"):] + repo = "file://%s" % os.path.abspath(path) + elif "://" not in repo: + repo = os.path.abspath(repo) + return repo + + +def make_hg_url(hgHost, repoPath, protocol='https', revision=None, + filename=None): + """construct a valid hg url from a base hg url (hg.mozilla.org), + repoPath, revision and possible filename""" + base = '%s://%s' % (protocol, hgHost) + repo = '/'.join(p.strip('/') for p in [base, repoPath]) + if not filename: + if not revision: + return repo + else: + return '/'.join([p.strip('/') for p in [repo, 'rev', revision]]) + else: + assert revision + return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]]) + + +def get_repo_name(repo): + return repo.rstrip('/').split('/')[-1] + + +def get_repo_path(repo): + repo = _make_absolute(repo) + if repo.startswith("/"): + return repo.lstrip("/") + else: + return urlsplit(repo).path.lstrip("/") + + +def get_revision(path): + """Returns which revision directory `path` currently has checked out.""" + return get_output(['hg', 'parent', '--template', '{node|short}'], cwd=path) + + +def get_branch(path): + return get_output(['hg', 'branch'], cwd=path).strip() + + +def get_branches(path): + branches = [] + for line in get_output(['hg', 'branches', '-c'], cwd=path).splitlines(): + branches.append(line.split()[0]) + return branches + + +def is_hg_cset(rev): + """Retruns True if passed revision represents a valid HG revision + (long or short(er) 40 bit hex)""" + try: + int(rev, 16) + return True + except (TypeError, ValueError): + return False + + +def hg_ver(): + """Returns the current version of hg, as a tuple of + (major, minor, build)""" + ver_string = get_output(['hg', '-q', 'version']) + match = re.search("\(version ([0-9.]+)\)", ver_string) + if match: + bits = match.group(1).split(".") + if len(bits) < 3: + bits += (0,) + ver = tuple(int(b) for b in bits) + else: + ver = (0, 0, 0) + log.debug("Running hg version %s", ver) + return ver + + +def purge(dest): + """Purge the repository of all untracked and ignored files.""" + try: + run_cmd(['hg', '--config', 'extensions.purge=', 'purge', + '-a', '--all', dest], cwd=dest) + except subprocess.CalledProcessError, e: + log.debug('purge failed: %s' % e) + raise + + +def update(dest, branch=None, revision=None): + """Updates working copy `dest` to `branch` or `revision`. If neither is + set then the working copy will be updated to the latest revision on the + current branch. Local changes will be discarded.""" + # If we have a revision, switch to that + if revision is not None: + cmd = ['hg', 'update', '-C', '-r', revision] + run_cmd(cmd, cwd=dest) + else: + # Check & switch branch + local_branch = get_output(['hg', 'branch'], cwd=dest).strip() + + cmd = ['hg', 'update', '-C'] + + # If this is different, checkout the other branch + if branch and branch != local_branch: + cmd.append(branch) + + run_cmd(cmd, cwd=dest) + return get_revision(dest) + + +def clone(repo, dest, branch=None, revision=None, update_dest=True, + clone_by_rev=False, mirrors=None, bundles=None): + """Clones hg repo and places it at `dest`, replacing whatever else is + there. The working copy will be empty. + + If `revision` is set, only the specified revision and its ancestors will + be cloned. + + If `update_dest` is set, then `dest` will be updated to `revision` if + set, otherwise to `branch`, otherwise to the head of default. + + If `mirrors` is set, will try and clone from the mirrors before + cloning from `repo`. + + If `bundles` is set, will try and download the bundle first and + unbundle it. If successful, will pull in new revisions from mirrors or + the master repo. If unbundling fails, will fall back to doing a regular + clone from mirrors or the master repo. + + Regardless of how the repository ends up being cloned, the 'default' path + will point to `repo`. + """ + if os.path.exists(dest): + remove_path(dest) + + if bundles: + log.info("Attempting to initialize clone with bundles") + for bundle in bundles: + if os.path.exists(dest): + remove_path(dest) + init(dest) + log.info("Trying to use bundle %s", bundle) + try: + if not unbundle(bundle, dest): + remove_path(dest) + continue + adjust_paths(dest, default=repo) + # Now pull / update + return pull(repo, dest, update_dest=update_dest, + mirrors=mirrors, revision=revision, branch=branch) + except Exception: + remove_path(dest) + log.exception("Problem unbundling/pulling from %s", bundle) + continue + else: + log.info("Using bundles failed; falling back to clone") + + if mirrors: + log.info("Attempting to clone from mirrors") + for mirror in mirrors: + log.info("Cloning from %s", mirror) + try: + retval = clone(mirror, dest, branch, revision, + update_dest=update_dest, clone_by_rev=clone_by_rev) + adjust_paths(dest, default=repo) + return retval + except: + log.exception("Problem cloning from mirror %s", mirror) + continue + else: + log.info("Pulling from mirrors failed; falling back to %s", repo) + # We may have a partial repo here; mercurial() copes with that + # We need to make sure our paths are correct though + if os.path.exists(os.path.join(dest, '.hg')): + adjust_paths(dest, default=repo) + return mercurial(repo, dest, branch, revision, autoPurge=True, + update_dest=update_dest, clone_by_rev=clone_by_rev) + + cmd = ['hg', 'clone'] + if not update_dest: + cmd.append('-U') + + if clone_by_rev: + if revision: + cmd.extend(['-r', revision]) + elif branch: + # hg >= 1.6 supports -b branch for cloning + ver = hg_ver() + if ver >= (1, 6, 0): + cmd.extend(['-b', branch]) + + cmd.extend([repo, dest]) + run_cmd(cmd) + + if update_dest: + return update(dest, branch, revision) + + +def common_args(revision=None, branch=None, ssh_username=None, ssh_key=None): + """Fill in common hg arguments, encapsulating logic checks that depend on + mercurial versions and provided arguments""" + args = [] + if ssh_username or ssh_key: + opt = ['-e', 'ssh'] + if ssh_username: + opt[1] += ' -l %s' % ssh_username + if ssh_key: + opt[1] += ' -i %s' % ssh_key + args.extend(opt) + if revision: + args.extend(['-r', revision]) + elif branch: + if hg_ver() >= (1, 6, 0): + args.extend(['-b', branch]) + return args + + +def pull(repo, dest, update_dest=True, mirrors=None, **kwargs): + """Pulls changes from hg repo and places it in `dest`. + + If `update_dest` is set, then `dest` will be updated to `revision` if + set, otherwise to `branch`, otherwise to the head of default. + + If `mirrors` is set, will try and pull from the mirrors first before + `repo`.""" + + if mirrors: + for mirror in mirrors: + try: + return pull(mirror, dest, update_dest=update_dest, **kwargs) + except: + log.exception("Problem pulling from mirror %s", mirror) + continue + else: + log.info("Pulling from mirrors failed; falling back to %s", repo) + + # Convert repo to an absolute path if it's a local repository + repo = _make_absolute(repo) + cmd = ['hg', 'pull'] + # Don't pass -r to "hg pull", except when it's a valid HG revision. + # Pulling using tag names is dangerous: it uses the local .hgtags, so if + # the tag has moved on the remote side you won't pull the new revision the + # remote tag refers to. + pull_kwargs = kwargs.copy() + if 'revision' in pull_kwargs and \ + not is_hg_cset(pull_kwargs['revision']): + del pull_kwargs['revision'] + + cmd.extend(common_args(**pull_kwargs)) + + cmd.append(repo) + run_cmd(cmd, cwd=dest) + + if update_dest: + branch = None + if 'branch' in kwargs and kwargs['branch']: + branch = kwargs['branch'] + revision = None + if 'revision' in kwargs and kwargs['revision']: + revision = kwargs['revision'] + return update(dest, branch=branch, revision=revision) + +# Defines the places of attributes in the tuples returned by `out' +REVISION, BRANCH = 0, 1 + + +def out(src, remote, **kwargs): + """Check for outgoing changesets present in a repo""" + cmd = ['hg', '-q', 'out', '--template', '{node} {branches}\n'] + cmd.extend(common_args(**kwargs)) + cmd.append(remote) + if os.path.exists(src): + try: + revs = [] + for line in get_output(cmd, cwd=src).rstrip().split("\n"): + try: + rev, branch = line.split() + # Mercurial displays no branch at all if the revision is on + # "default" + except ValueError: + rev = line.rstrip() + branch = "default" + revs.append((rev, branch)) + return revs + except subprocess.CalledProcessError, inst: + # In some situations, some versions of Mercurial return "1" + # if no changes are found, so we need to ignore this return code + if inst.returncode == 1: + return [] + raise + + +def push(src, remote, push_new_branches=True, force=False, **kwargs): + cmd = ['hg', 'push'] + cmd.extend(common_args(**kwargs)) + if force: + cmd.append('-f') + if push_new_branches: + cmd.append('--new-branch') + cmd.append(remote) + run_cmd(cmd, cwd=src) + + +def mercurial(repo, dest, branch=None, revision=None, update_dest=True, + shareBase=DefaultShareBase, allowUnsharedLocalClones=False, + clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False): + """Makes sure that `dest` is has `revision` or `branch` checked out from + `repo`. + + Do what it takes to make that happen, including possibly clobbering + dest. + + If allowUnsharedLocalClones is True and we're trying to use the share + extension but fail, then we will be able to clone from the shared repo to + our destination. If this is False, the default, then if we don't have the + share extension we will just clone from the remote repository. + + If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'. + This is slower, but useful when cloning repos with lots of heads. + + If `mirrors` is set, will try and use the mirrors before `repo`. + + If `bundles` is set, will try and download the bundle first and + unbundle it instead of doing a full clone. If successful, will pull in + new revisions from mirrors or the master repo. If unbundling fails, will + fall back to doing a regular clone from mirrors or the master repo. + """ + dest = os.path.abspath(dest) + if shareBase is DefaultShareBase: + shareBase = os.environ.get("HG_SHARE_BASE_DIR", None) + + log.info("Reporting hg version in use") + cmd = ['hg', '-q', 'version'] + run_cmd(cmd, cwd='.') + + if shareBase: + # Check that 'hg share' works + try: + log.info("Checking if share extension works") + output = get_output(['hg', 'help', 'share'], dont_log=True) + if 'no commands defined' in output: + # Share extension is enabled, but not functional + log.info("Disabling sharing since share extension doesn't seem to work (1)") + shareBase = None + elif 'unknown command' in output: + # Share extension is disabled + log.info("Disabling sharing since share extension doesn't seem to work (2)") + shareBase = None + except subprocess.CalledProcessError: + # The command failed, so disable sharing + log.info("Disabling sharing since share extension doesn't seem to work (3)") + shareBase = None + + # Check that our default path is correct + if os.path.exists(os.path.join(dest, '.hg')): + hgpath = path(dest, "default") + + # Make sure that our default path is correct + if hgpath != _make_absolute(repo): + log.info("hg path isn't correct (%s should be %s); clobbering", + hgpath, _make_absolute(repo)) + remove_path(dest) + + # If the working directory already exists and isn't using share we update + # the working directory directly from the repo, ignoring the sharing + # settings + if os.path.exists(dest): + if not os.path.exists(os.path.join(dest, ".hg")): + log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest) + remove_path(dest) + elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")): + try: + if autoPurge: + purge(dest) + return pull(repo, dest, update_dest=update_dest, branch=branch, + revision=revision, + mirrors=mirrors) + except subprocess.CalledProcessError: + log.warning("Error pulling changes into %s from %s; clobbering", dest, repo) + log.debug("Exception:", exc_info=True) + remove_path(dest) + + # If that fails for any reason, and sharing is requested, we'll try to + # update the shared repository, and then update the working directory from + # that. + if shareBase: + sharedRepo = os.path.join(shareBase, get_repo_path(repo)) + dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath') + + if os.path.exists(sharedRepo): + hgpath = path(sharedRepo, "default") + + # Make sure that our default path is correct + if hgpath != _make_absolute(repo): + log.info("hg path isn't correct (%s should be %s); clobbering", + hgpath, _make_absolute(repo)) + # we need to clobber both the shared checkout and the dest, + # since hgrc needs to be in both places + remove_path(sharedRepo) + remove_path(dest) + + if os.path.exists(dest_sharedPath): + # Make sure that the sharedpath points to sharedRepo + dest_sharedPath_data = os.path.normpath( + open(dest_sharedPath).read()) + norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg')) + if dest_sharedPath_data != norm_sharedRepo: + # Clobber! + log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering", + dest_sharedPath_data, repo, norm_sharedRepo) + remove_path(dest) + + try: + log.info("Updating shared repo") + mercurial(repo, sharedRepo, branch=branch, revision=revision, + update_dest=False, shareBase=None, clone_by_rev=clone_by_rev, + mirrors=mirrors, bundles=bundles, autoPurge=False) + if os.path.exists(dest): + if autoPurge: + purge(dest) + return update(dest, branch=branch, revision=revision) + + try: + log.info("Trying to share %s to %s", sharedRepo, dest) + return share(sharedRepo, dest, branch=branch, revision=revision) + except subprocess.CalledProcessError: + if not allowUnsharedLocalClones: + # Re-raise the exception so it gets caught below. + # We'll then clobber dest, and clone from original repo + raise + + log.warning("Error calling hg share from %s to %s;" + "falling back to normal clone from shared repo", + sharedRepo, dest) + # Do a full local clone first, and then update to the + # revision we want + # This lets us use hardlinks for the local clone if the OS + # supports it + clone(sharedRepo, dest, update_dest=False, + mirrors=mirrors, bundles=bundles) + return update(dest, branch=branch, revision=revision) + except subprocess.CalledProcessError: + log.warning( + "Error updating %s from sharedRepo (%s): ", dest, sharedRepo) + log.debug("Exception:", exc_info=True) + remove_path(dest) + # end if shareBase + + if not os.path.exists(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + + # Share isn't available or has failed, clone directly from the source + return clone(repo, dest, branch, revision, + update_dest=update_dest, mirrors=mirrors, + bundles=bundles, clone_by_rev=clone_by_rev) + + +def apply_and_push(localrepo, remote, changer, max_attempts=10, + ssh_username=None, ssh_key=None, force=False): + """This function calls `changer' to make changes to the repo, and tries + its hardest to get them to the origin repo. `changer' must be a + callable object that receives two arguments: the directory of the local + repository, and the attempt number. This function will push ALL + changesets missing from remote.""" + assert callable(changer) + branch = get_branch(localrepo) + changer(localrepo, 1) + for n in range(1, max_attempts + 1): + new_revs = [] + try: + new_revs = out(src=localrepo, remote=remote, + ssh_username=ssh_username, + ssh_key=ssh_key) + if len(new_revs) < 1: + raise HgUtilError("No revs to push") + push(src=localrepo, remote=remote, ssh_username=ssh_username, + ssh_key=ssh_key, force=force) + return + except subprocess.CalledProcessError, e: + log.debug("Hit error when trying to push: %s" % str(e)) + if n == max_attempts: + log.debug("Tried %d times, giving up" % max_attempts) + for r in reversed(new_revs): + run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n', + r[REVISION]], cwd=localrepo) + raise HgUtilError("Failed to push") + pull(remote, localrepo, update_dest=False, + ssh_username=ssh_username, ssh_key=ssh_key) + # After we successfully rebase or strip away heads the push is + # is attempted again at the start of the loop + try: + run_cmd(['hg', '--config', 'ui.merge=internal:merge', + 'rebase'], cwd=localrepo) + except subprocess.CalledProcessError, e: + log.debug("Failed to rebase: %s" % str(e)) + update(localrepo, branch=branch) + for r in reversed(new_revs): + run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n', + r[REVISION]], cwd=localrepo) + changer(localrepo, n + 1) + + +def share(source, dest, branch=None, revision=None): + """Creates a new working directory in "dest" that shares history with + "source" using Mercurial's share extension""" + run_cmd(['hg', 'share', '-U', source, dest]) + return update(dest, branch=branch, revision=revision) + + +def cleanOutgoingRevs(reponame, remote, username, sshKey): + outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, + ssh_username=username, + ssh_key=sshKey)) + for r in reversed(outgoingRevs): + run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n', + r[REVISION]], cwd=reponame) + + +def path(src, name='default'): + """Returns the remote path associated with "name" """ + try: + return get_output(['hg', 'path', name], cwd=src).strip() + except subprocess.CalledProcessError: + return None + + +def init(dest): + """Initializes an empty repo in `dest`""" + run_cmd(['hg', 'init', dest]) + + +def unbundle(bundle, dest): + """Unbundles the bundle located at `bundle` into `dest`. + + `bundle` can be a local file or remote url.""" + try: + get_output(['hg', 'unbundle', bundle], cwd=dest, include_stderr=True) + return True + except subprocess.CalledProcessError: + return False + + +def adjust_paths(dest, **paths): + """Adjusts paths in `dest`/.hg/hgrc so that names in `paths` are set to + paths[name]. + + Note that any comments in the hgrc will be lost if changes are made to the + file.""" + hgrc = os.path.join(dest, '.hg', 'hgrc') + config = RawConfigParser() + config.read(hgrc) + + if not config.has_section('paths'): + config.add_section('paths') + + changed = False + for path_name, path_value in paths.items(): + if (not config.has_option('paths', path_name) or + config.get('paths', path_name) != path_value): + changed = True + config.set('paths', path_name, path_value) + + if changed: + config.write(open(hgrc, 'w')) + + +def commit(dest, msg, user=None): + cmd = ['hg', 'commit', '-m', msg] + if user: + cmd.extend(['-u', user]) + run_cmd(cmd, cwd=dest) + return get_revision(dest) + + +def tag(dest, tags, user=None, msg=None, rev=None, force=None): + cmd = ['hg', 'tag'] + if user: + cmd.extend(['-u', user]) + if msg: + cmd.extend(['-m', msg]) + if rev: + cmd.extend(['-r', rev]) + if force: + cmd.append('-f') + cmd.extend(tags) + run_cmd(cmd, cwd=dest) + return get_revision(dest) |