diff options
Diffstat (limited to 'testing/mozbase/manifestparser/manifestparser/filters.py')
| -rw-r--r-- | testing/mozbase/manifestparser/manifestparser/filters.py | 421 |
1 files changed, 421 insertions, 0 deletions
diff --git a/testing/mozbase/manifestparser/manifestparser/filters.py b/testing/mozbase/manifestparser/manifestparser/filters.py new file mode 100644 index 000000000..e832c0da6 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/filters.py @@ -0,0 +1,421 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A filter is a callable that accepts an iterable of test objects and a +dictionary of values, and returns a new iterable of test objects. It is +possible to define custom filters if the built-in ones are not enough. +""" + +from collections import defaultdict, MutableSequence +import itertools +import os + +from .expression import ( + parse, + ParseError, +) + + +# built-in filters + +def skip_if(tests, values): + """ + Sets disabled on all tests containing the `skip-if` tag and whose condition + is True. This filter is added by default. + """ + tag = 'skip-if' + for test in tests: + if tag in test and parse(test[tag], **values): + test.setdefault('disabled', '{}: {}'.format(tag, test[tag])) + yield test + + +def run_if(tests, values): + """ + Sets disabled on all tests containing the `run-if` tag and whose condition + is False. This filter is added by default. + """ + tag = 'run-if' + for test in tests: + if tag in test and not parse(test[tag], **values): + test.setdefault('disabled', '{}: {}'.format(tag, test[tag])) + yield test + + +def fail_if(tests, values): + """ + Sets expected to 'fail' on all tests containing the `fail-if` tag and whose + condition is True. This filter is added by default. + """ + tag = 'fail-if' + for test in tests: + if tag in test and parse(test[tag], **values): + test['expected'] = 'fail' + yield test + + +def enabled(tests, values): + """ + Removes all tests containing the `disabled` key. This filter can be + added by passing `disabled=False` into `active_tests`. + """ + for test in tests: + if 'disabled' not in test: + yield test + + +def exists(tests, values): + """ + Removes all tests that do not exist on the file system. This filter is + added by default, but can be removed by passing `exists=False` into + `active_tests`. + """ + for test in tests: + if os.path.exists(test['path']): + yield test + + +# built-in instance filters + +class InstanceFilter(object): + """ + Generally only one instance of a class filter should be applied at a time. + Two instances of `InstanceFilter` are considered equal if they have the + same class name. This ensures only a single instance is ever added to + `filterlist`. This class also formats filters' __str__ method for easier + debugging. + """ + unique = True + + def __init__(self, *args, **kwargs): + self.fmt_args = ', '.join(itertools.chain( + [str(a) for a in args], + ['{}={}'.format(k, v) for k, v in kwargs.iteritems()])) + + def __eq__(self, other): + if self.unique: + return self.__class__ == other.__class__ + return self.__hash__() == other.__hash__() + + def __str__(self): + return "{}({})".format(self.__class__.__name__, self.fmt_args) + + +class subsuite(InstanceFilter): + """ + If `name` is None, removes all tests that have a `subsuite` key. + Otherwise removes all tests that do not have a subsuite matching `name`. + + It is possible to specify conditional subsuite keys using: + subsuite = foo,condition + + where 'foo' is the subsuite name, and 'condition' is the same type of + condition used for skip-if. If the condition doesn't evaluate to true, + the subsuite designation will be removed from the test. + + :param name: The name of the subsuite to run (default None) + """ + + def __init__(self, name=None): + InstanceFilter.__init__(self, name=name) + self.name = name + + def __call__(self, tests, values): + # Look for conditional subsuites, and replace them with the subsuite + # itself (if the condition is true), or nothing. + for test in tests: + subsuite = test.get('subsuite', '') + if ',' in subsuite: + try: + subsuite, cond = subsuite.split(',') + except ValueError: + raise ParseError("subsuite condition can't contain commas") + matched = parse(cond, **values) + if matched: + test['subsuite'] = subsuite + else: + test['subsuite'] = '' + + # Filter on current subsuite + if self.name is None: + if not test.get('subsuite'): + yield test + else: + if test.get('subsuite', '') == self.name: + yield test + + +class chunk_by_slice(InstanceFilter): + """ + Basic chunking algorithm that splits tests evenly across total chunks. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param disabled: Whether to include disabled tests in the chunking + algorithm. If False, each chunk contains an equal number + of non-disabled tests. If True, each chunk contains an + equal number of tests (default False) + """ + + def __init__(self, this_chunk, total_chunks, disabled=False): + assert 1 <= this_chunk <= total_chunks + InstanceFilter.__init__(self, this_chunk, total_chunks, + disabled=disabled) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.disabled = disabled + + def __call__(self, tests, values): + tests = list(tests) + if self.disabled: + chunk_tests = tests[:] + else: + chunk_tests = [t for t in tests if 'disabled' not in t] + + tests_per_chunk = float(len(chunk_tests)) / self.total_chunks + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + if not self.disabled: + # map start and end back onto original list of tests. Disabled + # tests will still be included in the returned list, but each + # chunk will contain an equal number of enabled tests. + if self.this_chunk == 1: + start = 0 + elif start < len(chunk_tests): + start = tests.index(chunk_tests[start]) + + if self.this_chunk == self.total_chunks: + end = len(tests) + elif end < len(chunk_tests): + end = tests.index(chunk_tests[end]) + return (t for t in tests[start:end]) + + +class chunk_by_dir(InstanceFilter): + """ + Basic chunking algorithm that splits directories of tests evenly at a + given depth. + + For example, a depth of 2 means all test directories two path nodes away + from the base are gathered, then split evenly across the total number of + chunks. The number of tests in each of the directories is not taken into + account (so chunks will not contain an even number of tests). All test + paths must be relative to the same root (typically the root of the source + repository). + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param depth: the minimum depth of a subdirectory before it will be + considered unique + """ + + def __init__(self, this_chunk, total_chunks, depth): + InstanceFilter.__init__(self, this_chunk, total_chunks, depth) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.depth = depth + + def __call__(self, tests, values): + tests_by_dir = defaultdict(list) + ordered_dirs = [] + for test in tests: + path = test['relpath'] + + if path.startswith(os.sep): + path = path[1:] + + dirs = path.split(os.sep) + dirs = dirs[:min(self.depth, len(dirs) - 1)] + path = os.sep.join(dirs) + + # don't count directories that only have disabled tests in them, + # but still yield disabled tests that are alongside enabled tests + if path not in ordered_dirs and 'disabled' not in test: + ordered_dirs.append(path) + tests_by_dir[path].append(test) + + tests_per_chunk = float(len(ordered_dirs)) / self.total_chunks + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + for i in range(start, end): + for test in tests_by_dir.pop(ordered_dirs[i]): + yield test + + # find directories that only contain disabled tests. They still need to + # be yielded for reporting purposes. Put them all in chunk 1 for + # simplicity. + if self.this_chunk == 1: + disabled_dirs = [v for k, v in tests_by_dir.iteritems() + if k not in ordered_dirs] + for disabled_test in itertools.chain(*disabled_dirs): + yield disabled_test + + +class chunk_by_runtime(InstanceFilter): + """ + Chunking algorithm that attempts to group tests into chunks based on their + average runtimes. It keeps manifests of tests together and pairs slow + running manifests with fast ones. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param runtimes: dictionary of test runtime data, of the form + {<test path>: <average runtime>} + :param default_runtime: value in seconds to assign tests that don't exist + in the runtimes file + """ + + def __init__(self, this_chunk, total_chunks, runtimes, default_runtime=0): + InstanceFilter.__init__(self, this_chunk, total_chunks, runtimes, + default_runtime=default_runtime) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + + # defaultdict(lambda:<int>) assigns all non-existent keys the value of + # <int>. This means all tests we encounter that don't exist in the + # runtimes file will be assigned `default_runtime`. + self.runtimes = defaultdict(lambda: default_runtime) + self.runtimes.update(runtimes) + + def __call__(self, tests, values): + tests = list(tests) + manifests = set(t['manifest'] for t in tests) + + def total_runtime(tests): + return sum(self.runtimes[t['relpath']] for t in tests + if 'disabled' not in t) + + tests_by_manifest = [] + for manifest in manifests: + mtests = [t for t in tests if t['manifest'] == manifest] + tests_by_manifest.append((total_runtime(mtests), mtests)) + tests_by_manifest.sort(reverse=True) + + tests_by_chunk = [[0, []] for i in range(self.total_chunks)] + for runtime, batch in tests_by_manifest: + # sort first by runtime, then by number of tests in case of a tie. + # This guarantees the chunk with the fastest runtime will always + # get the next batch of tests. + tests_by_chunk.sort(key=lambda x: (x[0], len(x[1]))) + tests_by_chunk[0][0] += runtime + tests_by_chunk[0][1].extend(batch) + + return (t for t in tests_by_chunk[self.this_chunk - 1][1]) + + +class tags(InstanceFilter): + """ + Removes tests that don't contain any of the given tags. This overrides + InstanceFilter's __eq__ method, so multiple instances can be added. + Multiple tag filters is equivalent to joining tags with the AND operator. + + To define a tag in a manifest, add a `tags` attribute to a test or DEFAULT + section. Tests can have multiple tags, in which case they should be + whitespace delimited. For example: + + [test_foobar.html] + tags = foo bar + + :param tags: A tag or list of tags to filter tests on + """ + unique = False + + def __init__(self, tags): + InstanceFilter.__init__(self, tags) + if isinstance(tags, basestring): + tags = [tags] + self.tags = tags + + def __call__(self, tests, values): + for test in tests: + if 'tags' not in test: + continue + + test_tags = [t.strip() for t in test['tags'].split()] + if any(t in self.tags for t in test_tags): + yield test + + +class pathprefix(InstanceFilter): + """ + Removes tests that don't start with any of the given test paths. + + :param paths: A list of test paths to filter on + """ + + def __init__(self, paths): + InstanceFilter.__init__(self, paths) + if isinstance(paths, basestring): + paths = [paths] + self.paths = paths + + def __call__(self, tests, values): + for test in tests: + for tp in self.paths: + tp = os.path.normpath(tp) + + path = test['relpath'] + if os.path.isabs(tp): + path = test['path'] + + if not os.path.normpath(path).startswith(tp): + continue + + # any test path that points to a single file will be run no + # matter what, even if it's disabled + if 'disabled' in test and os.path.normpath(test['relpath']) == tp: + del test['disabled'] + yield test + break + + +# filter container + +DEFAULT_FILTERS = ( + skip_if, + run_if, + fail_if, +) +""" +By default :func:`~.active_tests` will run the :func:`~.skip_if`, +:func:`~.run_if` and :func:`~.fail_if` filters. +""" + + +class filterlist(MutableSequence): + """ + A MutableSequence that raises TypeError when adding a non-callable and + ValueError if the item is already added. + """ + + def __init__(self, items=None): + self.items = [] + if items: + self.items = list(items) + + def _validate(self, item): + if not callable(item): + raise TypeError("Filters must be callable!") + if item in self: + raise ValueError("Filter {} is already applied!".format(item)) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self._validate(value) + self.items[key] = value + + def __delitem__(self, key): + del self.items[key] + + def __len__(self): + return len(self.items) + + def insert(self, index, value): + self._validate(value) + self.items.insert(index, value) |