diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/reduce.py')
| -rw-r--r-- | testing/web-platform/harness/wptrunner/reduce.py | 197 |
1 files changed, 197 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/reduce.py b/testing/web-platform/harness/wptrunner/reduce.py new file mode 100644 index 000000000..c4487b9d3 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/reduce.py @@ -0,0 +1,197 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import tempfile +from cStringIO import StringIO +from collections import defaultdict + +import wptrunner +import wpttest + +from mozlog import commandline, reader + +logger = None + + +def setup_logging(args, defaults): + global logger + logger = commandline.setup_logging("web-platform-tests-unstable", args, defaults) + wptrunner.setup_stdlib_logger() + + for name in args.keys(): + if name.startswith("log_"): + args.pop(name) + + return logger + + +def group(items, size): + rv = [] + i = 0 + while i < len(items): + rv.append(items[i:i + size]) + i += size + + return rv + + +def next_power_of_two(num): + rv = 1 + while rv < num: + rv = rv << 1 + return rv + + +class Reducer(object): + def __init__(self, target, **kwargs): + self.target = target + + self.test_type = kwargs["test_types"][0] + run_info = wpttest.get_run_info(kwargs["metadata_root"], + kwargs["product"], + debug=False) + test_filter = wptrunner.TestFilter(include=kwargs["include"]) + self.test_loader = wptrunner.TestLoader(kwargs["tests_root"], + kwargs["metadata_root"], + [self.test_type], + run_info, + manifest_filer=test_filter) + if kwargs["repeat"] == 1: + logger.critical("Need to specify --repeat with more than one repetition") + sys.exit(1) + self.kwargs = kwargs + + def run(self): + all_tests = self.get_initial_tests() + + tests = all_tests[:-1] + target_test = [all_tests[-1]] + + if self.unstable(target_test): + return target_test + + if not self.unstable(all_tests): + return [] + + chunk_size = next_power_of_two(int(len(tests) / 2)) + logger.debug("Using chunk size %i" % chunk_size) + + while chunk_size >= 1: + logger.debug("%i tests remain" % len(tests)) + chunks = group(tests, chunk_size) + chunk_results = [None] * len(chunks) + + for i, chunk in enumerate(chunks): + logger.debug("Running chunk %i/%i of size %i" % (i + 1, len(chunks), chunk_size)) + trial_tests = [] + chunk_str = "" + for j, inc_chunk in enumerate(chunks): + if i != j and chunk_results[j] in (None, False): + chunk_str += "+" + trial_tests.extend(inc_chunk) + else: + chunk_str += "-" + logger.debug("Using chunks %s" % chunk_str) + trial_tests.extend(target_test) + + chunk_results[i] = self.unstable(trial_tests) + + # if i == len(chunks) - 2 and all(item is False for item in chunk_results[:-1]): + # Dangerous? optimisation that if you got stability for 0..N-1 chunks + # it must be unstable with the Nth chunk + # chunk_results[i+1] = True + # continue + + new_tests = [] + keep_str = "" + for result, chunk in zip(chunk_results, chunks): + if not result: + keep_str += "+" + new_tests.extend(chunk) + else: + keep_str += "-" + + logger.debug("Keeping chunks %s" % keep_str) + + tests = new_tests + + chunk_size = int(chunk_size / 2) + + return tests + target_test + + def unstable(self, tests): + logger.debug("Running with %i tests" % len(tests)) + + self.test_loader.tests = {self.test_type: tests} + + stdout, stderr = sys.stdout, sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + + with tempfile.NamedTemporaryFile() as f: + args = self.kwargs.copy() + args["log_raw"] = [f] + args["capture_stdio"] = False + wptrunner.setup_logging(args, {}) + wptrunner.run_tests(test_loader=self.test_loader, **args) + wptrunner.logger.remove_handler(wptrunner.logger.handlers[0]) + is_unstable = self.log_is_unstable(f) + + sys.stdout, sys.stderr = stdout, stderr + + logger.debug("Result was unstable with chunk removed" + if is_unstable else "stable") + + return is_unstable + + def log_is_unstable(self, log_f): + log_f.seek(0) + + statuses = defaultdict(set) + + def handle_status(item): + if item["test"] == self.target: + statuses[item["subtest"]].add(item["status"]) + + def handle_end(item): + if item["test"] == self.target: + statuses[None].add(item["status"]) + + reader.each_log(reader.read(log_f), + {"test_status": handle_status, + "test_end": handle_end}) + + logger.debug(str(statuses)) + + if not statuses: + logger.error("Didn't get any useful output from wptrunner") + log_f.seek(0) + for item in reader.read(log_f): + logger.debug(item) + return None + + return any(len(item) > 1 for item in statuses.itervalues()) + + def get_initial_tests(self): + # Need to pass in arguments + + all_tests = self.test_loader.tests[self.test_type] + tests = [] + for item in all_tests: + tests.append(item) + if item.url == self.target: + break + + logger.debug("Starting with tests: %s" % ("\n".join(item.id for item in tests))) + + return tests + + +def do_reduce(**kwargs): + target = kwargs.pop("target") + reducer = Reducer(target, **kwargs) + + unstable_set = reducer.run() + return unstable_set |