summaryrefslogtreecommitdiffstats
path: root/testing/tools/autotry/autotry.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/tools/autotry/autotry.py')
-rw-r--r--testing/tools/autotry/autotry.py586
1 files changed, 586 insertions, 0 deletions
diff --git a/testing/tools/autotry/autotry.py b/testing/tools/autotry/autotry.py
new file mode 100644
index 000000000..7b0350b61
--- /dev/null
+++ b/testing/tools/autotry/autotry.py
@@ -0,0 +1,586 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import itertools
+import os
+import re
+import subprocess
+import sys
+import which
+
+from collections import defaultdict
+
+import ConfigParser
+
+
+def arg_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
+ parser.add_argument('-b', '--build', dest='builds', default='do',
+ help='Build types to run (d for debug, o for optimized).')
+ parser.add_argument('-p', '--platform', dest='platforms', action='append',
+ help='Platforms to run (required if not found in the environment as AUTOTRY_PLATFORM_HINT).')
+ parser.add_argument('-u', '--unittests', dest='tests', action='append',
+ help='Test suites to run in their entirety.')
+ parser.add_argument('-t', '--talos', dest='talos', action='append',
+ help='Talos suites to run.')
+ parser.add_argument('--tag', dest='tags', action='append',
+ help='Restrict tests to the given tag (may be specified multiple times).')
+ parser.add_argument('--and', action='store_true', dest='intersection',
+ help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments.')
+ parser.add_argument('--no-push', dest='push', action='store_false',
+ help='Do not push to try as a result of running this command (if '
+ 'specified this command will only print calculated try '
+ 'syntax and selection info).')
+ parser.add_argument('--save', dest='save', action='store',
+ help='Save the command line arguments for future use with --preset.')
+ parser.add_argument('--preset', dest='load', action='store',
+ help='Load a saved set of arguments. Additional arguments will override saved ones.')
+ parser.add_argument('--list', action='store_true',
+ help='List all saved try strings')
+ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False,
+ help='Print detailed information about the resulting test selection '
+ 'and commands performed.')
+ for arg, opts in AutoTry.pass_through_arguments.items():
+ parser.add_argument(arg, **opts)
+ return parser
+
+class TryArgumentTokenizer(object):
+ symbols = [("seperator", ","),
+ ("list_start", "\["),
+ ("list_end", "\]"),
+ ("item", "([^,\[\]\s][^,\[\]]+)"),
+ ("space", "\s+")]
+ token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
+
+ def tokenize(self, data):
+ for match in self.token_re.finditer(data):
+ symbol = match.lastgroup
+ data = match.group(symbol)
+ if symbol == "space":
+ pass
+ else:
+ yield symbol, data
+
+class TryArgumentParser(object):
+ """Simple three-state parser for handling expressions
+ of the from "foo[sub item, another], bar,baz". This takes
+ input from the TryArgumentTokenizer and runs through a small
+ state machine, returning a dictionary of {top-level-item:[sub_items]}
+ i.e. the above would result in
+ {"foo":["sub item", "another"], "bar": [], "baz": []}
+ In the case of invalid input a ValueError is raised."""
+
+ EOF = object()
+
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.tokens = None
+ self.current_item = None
+ self.data = {}
+ self.token = None
+ self.state = None
+
+ def parse(self, tokens):
+ self.reset()
+ self.tokens = tokens
+ self.consume()
+ self.state = self.item_state
+ while self.token[0] != self.EOF:
+ self.state()
+ return self.data
+
+ def consume(self):
+ try:
+ self.token = self.tokens.next()
+ except StopIteration:
+ self.token = (self.EOF, None)
+
+ def expect(self, *types):
+ if self.token[0] not in types:
+ raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
+
+ def item_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ if value not in self.data:
+ self.data[value] = []
+ self.current_item = value
+ self.consume()
+ if self.token[0] == "seperator":
+ self.consume()
+ elif self.token[0] == "list_start":
+ self.consume()
+ self.state = self.subitem_state
+ elif self.token[0] == self.EOF:
+ pass
+ else:
+ raise ValueError
+
+ def subitem_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ self.data[self.current_item].append(value)
+ self.consume()
+ if self.token[0] == "seperator":
+ self.consume()
+ elif self.token[0] == "list_end":
+ self.consume()
+ self.state = self.after_list_end_state
+ else:
+ raise ValueError
+
+ def after_list_end_state(self):
+ self.expect("seperator")
+ self.consume()
+ self.state = self.item_state
+
+def parse_arg(arg):
+ tokenizer = TryArgumentTokenizer()
+ parser = TryArgumentParser()
+ return parser.parse(tokenizer.tokenize(arg))
+
+class AutoTry(object):
+
+ # Maps from flavors to the job names needed to run that flavour
+ flavor_jobs = {
+ 'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+ 'xpcshell': ['xpcshell'],
+ 'chrome': ['mochitest-o'],
+ 'browser-chrome': ['mochitest-browser-chrome-1',
+ 'mochitest-e10s-browser-chrome-1'],
+ 'devtools-chrome': ['mochitest-devtools-chrome-1',
+ 'mochitest-e10s-devtools-chrome-1'],
+ 'crashtest': ['crashtest', 'crashtest-e10s'],
+ 'reftest': ['reftest', 'reftest-e10s'],
+ 'web-platform-tests': ['web-platform-tests-1'],
+ }
+
+ flavor_suites = {
+ "mochitest": "mochitests",
+ "xpcshell": "xpcshell",
+ "chrome": "mochitest-o",
+ "browser-chrome": "mochitest-bc",
+ "devtools-chrome": "mochitest-dt",
+ "crashtest": "crashtest",
+ "reftest": "reftest",
+ "web-platform-tests": "web-platform-tests",
+ }
+
+ compiled_suites = [
+ "cppunit",
+ "gtest",
+ "jittest",
+ ]
+
+ common_suites = [
+ "cppunit",
+ "crashtest",
+ "firefox-ui-functional",
+ "gtest",
+ "jittest",
+ "jsreftest",
+ "marionette",
+ "marionette-e10s",
+ "media-tests",
+ "mochitests",
+ "reftest",
+ "web-platform-tests",
+ "xpcshell",
+ ]
+
+ # Arguments we will accept on the command line and pass through to try
+ # syntax with no further intervention. The set is taken from
+ # http://trychooser.pub.build.mozilla.org with a few additions.
+ #
+ # Note that the meaning of store_false and store_true arguments is
+ # not preserved here, as we're only using these to echo the literal
+ # arguments to another consumer. Specifying either store_false or
+ # store_true here will have an equivalent effect.
+ pass_through_arguments = {
+ '--rebuild': {
+ 'action': 'store',
+ 'dest': 'rebuild',
+ 'help': 'Re-trigger all test jobs (up to 20 times)',
+ },
+ '--rebuild-talos': {
+ 'action': 'store',
+ 'dest': 'rebuild_talos',
+ 'help': 'Re-trigger all talos jobs',
+ },
+ '--interactive': {
+ 'action': 'store_true',
+ 'dest': 'interactive',
+ 'help': 'Allow ssh-like access to running test containers',
+ },
+ '--no-retry': {
+ 'action': 'store_true',
+ 'dest': 'no_retry',
+ 'help': 'Do not retrigger failed tests',
+ },
+ '--setenv': {
+ 'action': 'append',
+ 'dest': 'setenv',
+ 'help': 'Set the corresponding variable in the test environment for'
+ 'applicable harnesses.',
+ },
+ '-f': {
+ 'action': 'store_true',
+ 'dest': 'failure_emails',
+ 'help': 'Request failure emails only',
+ },
+ '--failure-emails': {
+ 'action': 'store_true',
+ 'dest': 'failure_emails',
+ 'help': 'Request failure emails only',
+ },
+ '-e': {
+ 'action': 'store_true',
+ 'dest': 'all_emails',
+ 'help': 'Request all emails',
+ },
+ '--all-emails': {
+ 'action': 'store_true',
+ 'dest': 'all_emails',
+ 'help': 'Request all emails',
+ },
+ '--artifact': {
+ 'action': 'store_true',
+ 'dest': 'artifact',
+ 'help': 'Force artifact builds where possible.',
+ }
+ }
+
+ def __init__(self, topsrcdir, resolver_func, mach_context):
+ self.topsrcdir = topsrcdir
+ self._resolver_func = resolver_func
+ self._resolver = None
+ self.mach_context = mach_context
+
+ if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
+ self._use_git = False
+ else:
+ self._use_git = True
+
+ @property
+ def resolver(self):
+ if self._resolver is None:
+ self._resolver = self._resolver_func()
+ return self._resolver
+
+ @property
+ def config_path(self):
+ return os.path.join(self.mach_context.state_dir, "autotry.ini")
+
+ def load_config(self, name):
+ config = ConfigParser.RawConfigParser()
+ success = config.read([self.config_path])
+ if not success:
+ return None
+
+ try:
+ data = config.get("try", name)
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ return None
+
+ kwargs = vars(arg_parser().parse_args(self.split_try_string(data)))
+
+ return kwargs
+
+ def list_presets(self):
+ config = ConfigParser.RawConfigParser()
+ success = config.read([self.config_path])
+
+ data = []
+ if success:
+ try:
+ data = config.items("try")
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ pass
+
+ if not data:
+ print("No presets found")
+
+ for name, try_string in data:
+ print("%s: %s" % (name, try_string))
+
+ def split_try_string(self, data):
+ return re.findall(r'(?:\[.*?\]|\S)+', data)
+
+ def save_config(self, name, data):
+ assert data.startswith("try: ")
+ data = data[len("try: "):]
+
+ parser = ConfigParser.RawConfigParser()
+ parser.read([self.config_path])
+
+ if not parser.has_section("try"):
+ parser.add_section("try")
+
+ parser.set("try", name, data)
+
+ with open(self.config_path, "w") as f:
+ parser.write(f)
+
+ def paths_by_flavor(self, paths=None, tags=None):
+ paths_by_flavor = defaultdict(set)
+
+ if not (paths or tags):
+ return dict(paths_by_flavor)
+
+ tests = list(self.resolver.resolve_tests(paths=paths,
+ tags=tags))
+
+ for t in tests:
+ if t['flavor'] in self.flavor_suites:
+ flavor = t['flavor']
+ if 'subsuite' in t and t['subsuite'] == 'devtools':
+ flavor = 'devtools-chrome'
+
+ if flavor in ['crashtest', 'reftest']:
+ manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
+ paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
+ elif 'dir_relpath' in t:
+ paths_by_flavor[flavor].add(t['dir_relpath'])
+ else:
+ file_relpath = os.path.relpath(t['path'], self.topsrcdir)
+ dir_relpath = os.path.dirname(file_relpath)
+ paths_by_flavor[flavor].add(dir_relpath)
+
+ for flavor, path_set in paths_by_flavor.items():
+ paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
+
+ return dict(paths_by_flavor)
+
+ def deduplicate_prefixes(self, path_set, input_paths):
+ # Removes paths redundant to test selection in the given path set.
+ # If a path was passed on the commandline that is the prefix of a
+ # path in our set, we only need to include the specified prefix to
+ # run the intended tests (every test in "layout/base" will run if
+ # "layout" is passed to the reftest harness).
+ removals = set()
+ additions = set()
+
+ for path in path_set:
+ full_path = path
+ while path:
+ path, _ = os.path.split(path)
+ if path in input_paths:
+ removals.add(full_path)
+ additions.add(path)
+
+ return additions | (path_set - removals)
+
+ def remove_duplicates(self, paths_by_flavor, tests):
+ rv = {}
+ for item in paths_by_flavor:
+ if self.flavor_suites[item] not in tests:
+ rv[item] = paths_by_flavor[item].copy()
+ return rv
+
+ def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags,
+ extras, intersection):
+ parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
+
+ suites = tests if not intersection else {}
+ paths = set()
+ for flavor, flavor_tests in paths_by_flavor.iteritems():
+ suite = self.flavor_suites[flavor]
+ if suite not in suites and (not intersection or suite in tests):
+ for job_name in self.flavor_jobs[flavor]:
+ for test in flavor_tests:
+ paths.add("%s:%s" % (flavor, test))
+ suites[job_name] = tests.get(suite, [])
+
+ if not suites:
+ raise ValueError("No tests found matching filters")
+
+ if extras.get('artifact'):
+ rejected = []
+ for suite in suites.keys():
+ if any([suite.startswith(c) for c in self.compiled_suites]):
+ rejected.append(suite)
+ if rejected:
+ raise ValueError("You can't run {} with "
+ "--artifact option.".format(', '.join(rejected)))
+
+ parts.append("-u")
+ parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k,v in sorted(suites.items())) if suites else "none")
+
+ parts.append("-t")
+ parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k,v in sorted(talos.items())) if talos else "none")
+
+ if tags:
+ parts.append(' '.join('--tag %s' % t for t in tags))
+
+ if paths:
+ parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
+
+ args_by_dest = {v['dest']: k for k, v in AutoTry.pass_through_arguments.items()}
+ for dest, value in extras.iteritems():
+ assert dest in args_by_dest
+ arg = args_by_dest[dest]
+ action = AutoTry.pass_through_arguments[arg]['action']
+ if action == 'store':
+ parts.append(arg)
+ parts.append(value)
+ if action == 'append':
+ for e in value:
+ parts.append(arg)
+ parts.append(e)
+ if action in ('store_true', 'store_false'):
+ parts.append(arg)
+
+ try_syntax = " ".join(parts)
+ if extras.get('artifact') and 'all' in suites.keys():
+ message = ('You asked for |-u all| with |--artifact| but compiled-code tests ({tests})'
+ ' can\'t run against an artifact build. Try listing the suites you want'
+ ' instead. For example, this syntax covers most suites:\n{try_syntax}')
+ string_format = {
+ 'tests': ','.join(self.compiled_suites),
+ 'try_syntax': try_syntax.replace(
+ '-u all',
+ '-u ' + ','.join(sorted(set(self.common_suites) - set(self.compiled_suites)))
+ )
+ }
+ raise ValueError(message.format(**string_format))
+
+ return try_syntax
+
+ def _run_git(self, *args):
+ args = ['git'] + list(args)
+ ret = subprocess.call(args)
+ if ret:
+ print('ERROR git command %s returned %s' %
+ (args, ret))
+ sys.exit(1)
+
+ def _git_push_to_try(self, msg):
+ self._run_git('commit', '--allow-empty', '-m', msg)
+ try:
+ self._run_git('push', 'hg::ssh://hg.mozilla.org/try',
+ '+HEAD:refs/heads/branches/default/tip')
+ finally:
+ self._run_git('reset', 'HEAD~')
+
+ def _git_find_changed_files(self):
+ # This finds the files changed on the current branch based on the
+ # diff of the current branch its merge-base base with other branches.
+ try:
+ args = ['git', 'rev-parse', 'HEAD']
+ current_branch = subprocess.check_output(args).strip()
+ args = ['git', 'for-each-ref', 'refs/heads', 'refs/remotes',
+ '--format=%(objectname)']
+ all_branches = subprocess.check_output(args).splitlines()
+ other_branches = set(all_branches) - set([current_branch])
+ args = ['git', 'merge-base', 'HEAD'] + list(other_branches)
+ base_commit = subprocess.check_output(args).strip()
+ args = ['git', 'diff', '--name-only', '-z', 'HEAD', base_commit]
+ return subprocess.check_output(args).strip('\0').split('\0')
+ except subprocess.CalledProcessError as e:
+ print('Failed while determining files changed on this branch')
+ print('Failed whle running: %s' % args)
+ print(e.output)
+ sys.exit(1)
+
+ def _hg_find_changed_files(self):
+ hg_args = [
+ 'hg', 'log', '-r',
+ '::. and not public()',
+ '--template',
+ '{join(files, "\n")}\n',
+ ]
+ try:
+ return subprocess.check_output(hg_args).splitlines()
+ except subprocess.CalledProcessError as e:
+ print('Failed while finding files changed since the last '
+ 'public ancestor')
+ print('Failed whle running: %s' % hg_args)
+ print(e.output)
+ sys.exit(1)
+
+ def find_changed_files(self):
+ """Finds files changed in a local source tree.
+
+ For hg, changes since the last public ancestor of '.' are
+ considered. For git, changes in the current branch are considered.
+ """
+ if self._use_git:
+ return self._git_find_changed_files()
+ return self._hg_find_changed_files()
+
+ def push_to_try(self, msg, verbose):
+ if not self._use_git:
+ try:
+ hg_args = ['hg', 'push-to-try', '-m', msg]
+ subprocess.check_call(hg_args, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ print('ERROR hg command %s returned %s' % (hg_args, e.returncode))
+ print('\nmach failed to push to try. There may be a problem '
+ 'with your ssh key, or another issue with your mercurial '
+ 'installation.')
+ # Check for the presence of the "push-to-try" extension, and
+ # provide instructions if it can't be found.
+ try:
+ subprocess.check_output(['hg', 'showconfig',
+ 'extensions.push-to-try'])
+ except subprocess.CalledProcessError:
+ print('\nThe "push-to-try" hg extension is required. It '
+ 'can be installed to Mercurial 3.3 or above by '
+ 'running ./mach mercurial-setup')
+ sys.exit(1)
+ else:
+ try:
+ which.which('git-cinnabar')
+ self._git_push_to_try(msg)
+ except which.WhichError:
+ print('ERROR git-cinnabar is required to push from git to try with'
+ 'the autotry command.\n\nMore information can by found at '
+ 'https://github.com/glandium/git-cinnabar')
+ sys.exit(1)
+
+ def find_uncommited_changes(self):
+ if self._use_git:
+ stat = subprocess.check_output(['git', 'status', '-z'])
+ return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'D')
+ for entry in stat.split('\0'))
+ else:
+ stat = subprocess.check_output(['hg', 'status'])
+ return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'R')
+ for entry in stat.splitlines())
+
+ def find_paths_and_tags(self, verbose):
+ paths, tags = set(), set()
+ changed_files = self.find_changed_files()
+ if changed_files:
+ if verbose:
+ print("Pushing tests based on modifications to the "
+ "following files:\n\t%s" % "\n\t".join(changed_files))
+
+ from mozbuild.frontend.reader import (
+ BuildReader,
+ EmptyConfig,
+ )
+
+ config = EmptyConfig(self.topsrcdir)
+ reader = BuildReader(config)
+ files_info = reader.files_info(changed_files)
+
+ for path, info in files_info.items():
+ paths |= info.test_files
+ tags |= info.test_tags
+
+ if verbose:
+ if paths:
+ print("Pushing tests based on the following patterns:\n\t%s" %
+ "\n\t".join(paths))
+ if tags:
+ print("Pushing tests based on the following tags:\n\t%s" %
+ "\n\t".join(tags))
+ return paths, tags
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-26 08:10:11 +0000