diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/metadata.py')
| -rw-r--r-- | testing/web-platform/harness/wptrunner/metadata.py | 350 |
1 files changed, 350 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/metadata.py b/testing/web-platform/harness/wptrunner/metadata.py new file mode 100644 index 000000000..1a25898a9 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/metadata.py @@ -0,0 +1,350 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import sys +import tempfile +import types +import uuid +from collections import defaultdict + +from mozlog import reader +from mozlog import structuredlog + +import expected +import manifestupdate +import testloader +import wptmanifest +import wpttest +from vcs import git +manifest = None # Module that will be imported relative to test_root + +logger = structuredlog.StructuredLogger("web-platform-tests") + + +def load_test_manifests(serve_root, test_paths): + do_delayed_imports(serve_root) + manifest_loader = testloader.ManifestLoader(test_paths, False) + return manifest_loader.load() + + +def update_expected(test_paths, serve_root, log_file_names, + rev_old=None, rev_new="HEAD", ignore_existing=False, + sync_root=None, property_order=None, boolean_properties=None): + """Update the metadata files for web-platform-tests based on + the results obtained in a previous run""" + + manifests = load_test_manifests(serve_root, test_paths) + + change_data = {} + + if sync_root is not None: + if rev_old is not None: + rev_old = git("rev-parse", rev_old, repo=sync_root).strip() + rev_new = git("rev-parse", rev_new, repo=sync_root).strip() + + if rev_old is not None: + change_data = load_change_data(rev_old, rev_new, repo=sync_root) + + + expected_map_by_manifest = update_from_logs(manifests, + *log_file_names, + ignore_existing=ignore_existing, + property_order=property_order, + boolean_properties=boolean_properties) + + for test_manifest, expected_map in expected_map_by_manifest.iteritems(): + url_base = manifests[test_manifest]["url_base"] + metadata_path = test_paths[url_base]["metadata_path"] + write_changes(metadata_path, expected_map) + + results_changed = [item.test_path for item in expected_map.itervalues() if item.modified] + + return unexpected_changes(manifests, change_data, results_changed) + + +def do_delayed_imports(serve_root): + global manifest + from manifest import manifest + + +def files_in_repo(repo_root): + return git("ls-tree", "-r", "--name-only", "HEAD").split("\n") + + +def rev_range(rev_old, rev_new, symmetric=False): + joiner = ".." if not symmetric else "..." + return "".join([rev_old, joiner, rev_new]) + + +def paths_changed(rev_old, rev_new, repo): + data = git("diff", "--name-status", rev_range(rev_old, rev_new), repo=repo) + lines = [tuple(item.strip() for item in line.strip().split("\t", 1)) + for line in data.split("\n") if line.strip()] + output = set(lines) + return output + + +def load_change_data(rev_old, rev_new, repo): + changes = paths_changed(rev_old, rev_new, repo) + rv = {} + status_keys = {"M": "modified", + "A": "new", + "D": "deleted"} + # TODO: deal with renames + for item in changes: + rv[item[1]] = status_keys[item[0]] + return rv + + +def unexpected_changes(manifests, change_data, files_changed): + files_changed = set(files_changed) + + root_manifest = None + for manifest, paths in manifests.iteritems(): + if paths["url_base"] == "/": + root_manifest = manifest + break + else: + return [] + + rv = [] + + return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"] + +# For each testrun +# Load all files and scan for the suite_start entry +# Build a hash of filename: properties +# For each different set of properties, gather all chunks +# For each chunk in the set of chunks, go through all tests +# for each test, make a map of {conditionals: [(platform, new_value)]} +# Repeat for each platform +# For each test in the list of tests: +# for each conditional: +# If all the new values match (or there aren't any) retain that conditional +# If any new values mismatch mark the test as needing human attention +# Check if all the RHS values are the same; if so collapse the conditionals + + +def update_from_logs(manifests, *log_filenames, **kwargs): + ignore_existing = kwargs.get("ignore_existing", False) + property_order = kwargs.get("property_order") + boolean_properties = kwargs.get("boolean_properties") + + expected_map = {} + id_test_map = {} + + for test_manifest, paths in manifests.iteritems(): + expected_map_manifest, id_path_map_manifest = create_test_tree( + paths["metadata_path"], + test_manifest, + property_order=property_order, + boolean_properties=boolean_properties) + expected_map[test_manifest] = expected_map_manifest + id_test_map.update(id_path_map_manifest) + + updater = ExpectedUpdater(manifests, expected_map, id_test_map, + ignore_existing=ignore_existing) + for log_filename in log_filenames: + with open(log_filename) as f: + updater.update_from_log(f) + + for manifest_expected in expected_map.itervalues(): + for tree in manifest_expected.itervalues(): + for test in tree.iterchildren(): + for subtest in test.iterchildren(): + subtest.coalesce_expected() + test.coalesce_expected() + + return expected_map + +def directory_manifests(metadata_path): + rv = [] + for dirpath, dirname, filenames in os.walk(metadata_path): + if "__dir__.ini" in filenames: + rel_path = os.path.relpath(dirpath, metadata_path) + rv.append(os.path.join(rel_path, "__dir__.ini")) + return rv + +def write_changes(metadata_path, expected_map): + # First write the new manifest files to a temporary directory + temp_path = tempfile.mkdtemp(dir=os.path.split(metadata_path)[0]) + write_new_expected(temp_path, expected_map) + + # Keep all __dir__.ini files (these are not in expected_map because they + # aren't associated with a specific test) + keep_files = directory_manifests(metadata_path) + + # Copy all files in the root to the temporary location since + # these cannot be ini files + keep_files.extend(item for item in os.listdir(metadata_path) if + not os.path.isdir(os.path.join(metadata_path, item))) + + for item in keep_files: + dest_dir = os.path.dirname(os.path.join(temp_path, item)) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + shutil.copyfile(os.path.join(metadata_path, item), + os.path.join(temp_path, item)) + + # Then move the old manifest files to a new location + temp_path_2 = metadata_path + str(uuid.uuid4()) + os.rename(metadata_path, temp_path_2) + # Move the new files to the destination location and remove the old files + os.rename(temp_path, metadata_path) + shutil.rmtree(temp_path_2) + + +def write_new_expected(metadata_path, expected_map): + # Serialize the data back to a file + for tree in expected_map.itervalues(): + if not tree.is_empty: + manifest_str = wptmanifest.serialize(tree.node, skip_empty_data=True) + assert manifest_str != "" + path = expected.expected_path(metadata_path, tree.test_path) + dir = os.path.split(path)[0] + if not os.path.exists(dir): + os.makedirs(dir) + with open(path, "w") as f: + f.write(manifest_str) + + +class ExpectedUpdater(object): + def __init__(self, test_manifests, expected_tree, id_path_map, ignore_existing=False): + self.test_manifests = test_manifests + self.expected_tree = expected_tree + self.id_path_map = id_path_map + self.ignore_existing = ignore_existing + self.run_info = None + self.action_map = {"suite_start": self.suite_start, + "test_start": self.test_start, + "test_status": self.test_status, + "test_end": self.test_end} + self.tests_visited = {} + + self.test_cache = {} + + def update_from_log(self, log_file): + self.run_info = None + log_reader = reader.read(log_file) + reader.each_log(log_reader, self.action_map) + + def suite_start(self, data): + self.run_info = data["run_info"] + + def test_id(self, id): + if type(id) in types.StringTypes: + return id + else: + return tuple(id) + + def test_start(self, data): + test_id = self.test_id(data["test"]) + try: + test_manifest, test = self.id_path_map[test_id] + expected_node = self.expected_tree[test_manifest][test].get_test(test_id) + except KeyError: + print "Test not found %s, skipping" % test_id + return + self.test_cache[test_id] = expected_node + + if test_id not in self.tests_visited: + if self.ignore_existing: + expected_node.clear_expected() + self.tests_visited[test_id] = set() + + def test_status(self, data): + test_id = self.test_id(data["test"]) + test = self.test_cache.get(test_id) + if test is None: + return + test_cls = wpttest.manifest_test_cls[test.test_type] + + subtest = test.get_subtest(data["subtest"]) + + self.tests_visited[test.id].add(data["subtest"]) + + result = test_cls.subtest_result_cls( + data["subtest"], + data["status"], + data.get("message")) + + subtest.set_result(self.run_info, result) + + def test_end(self, data): + test_id = self.test_id(data["test"]) + test = self.test_cache.get(test_id) + if test is None: + return + test_cls = wpttest.manifest_test_cls[test.test_type] + + if data["status"] == "SKIP": + return + + result = test_cls.result_cls( + data["status"], + data.get("message")) + + test.set_result(self.run_info, result) + del self.test_cache[test_id] + + +def create_test_tree(metadata_path, test_manifest, property_order=None, + boolean_properties=None): + expected_map = {} + id_test_map = {} + exclude_types = frozenset(["stub", "helper", "manual"]) + include_types = set(manifest.item_types) - exclude_types + for test_path, tests in test_manifest.itertypes(*include_types): + expected_data = load_expected(test_manifest, metadata_path, test_path, tests, + property_order=property_order, + boolean_properties=boolean_properties) + if expected_data is None: + expected_data = create_expected(test_manifest, + test_path, + tests, + property_order=property_order, + boolean_properties=boolean_properties) + + for test in tests: + id_test_map[test.id] = (test_manifest, test) + expected_map[test] = expected_data + + return expected_map, id_test_map + + +def create_expected(test_manifest, test_path, tests, property_order=None, + boolean_properties=None): + expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base, + property_order=property_order, + boolean_properties=boolean_properties) + for test in tests: + expected.append(manifestupdate.TestNode.create(test.item_type, test.id)) + return expected + + +def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None, + boolean_properties=None): + expected_manifest = manifestupdate.get_manifest(metadata_path, + test_path, + test_manifest.url_base, + property_order=property_order, + boolean_properties=boolean_properties) + if expected_manifest is None: + return + + tests_by_id = {item.id: item for item in tests} + + # Remove expected data for tests that no longer exist + for test in expected_manifest.iterchildren(): + if not test.id in tests_by_id: + test.remove() + + # Add tests that don't have expected data + for test in tests: + if not expected_manifest.has_test(test.id): + expected_manifest.append(manifestupdate.TestNode.create(test.item_type, test.id)) + + return expected_manifest |