diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/wpttest.py')
| -rw-r--r-- | testing/web-platform/harness/wptrunner/wpttest.py | 351 |
1 files changed, 351 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/wpttest.py b/testing/web-platform/harness/wptrunner/wpttest.py new file mode 100644 index 000000000..177e9208a --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wpttest.py @@ -0,0 +1,351 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DEFAULT_TIMEOUT = 10 # seconds +LONG_TIMEOUT = 60 # seconds + +import os + +import mozinfo + +from wptmanifest.parser import atoms + +atom_reset = atoms["Reset"] +enabled_tests = set(["testharness", "reftest", "wdspec"]) + + +class Result(object): + def __init__(self, status, message, expected=None, extra=None): + if status not in self.statuses: + raise ValueError("Unrecognised status %s" % status) + self.status = status + self.message = message + self.expected = expected + self.extra = extra + + def __repr__(self): + return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status) + + +class SubtestResult(object): + def __init__(self, name, status, message, stack=None, expected=None): + self.name = name + if status not in self.statuses: + raise ValueError("Unrecognised status %s" % status) + self.status = status + self.message = message + self.stack = stack + self.expected = expected + + def __repr__(self): + return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status) + + +class TestharnessResult(Result): + default_expected = "OK" + statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) + + +class TestharnessSubtestResult(SubtestResult): + default_expected = "PASS" + statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"]) + + +class ReftestResult(Result): + default_expected = "PASS" + statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) + + +class WdspecResult(Result): + default_expected = "OK" + statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) + + +class WdspecSubtestResult(SubtestResult): + default_expected = "PASS" + statuses = set(["PASS", "FAIL", "ERROR"]) + + +def get_run_info(metadata_root, product, **kwargs): + if product == "b2g": + return B2GRunInfo(metadata_root, product, **kwargs) + else: + return RunInfo(metadata_root, product, **kwargs) + + +class RunInfo(dict): + def __init__(self, metadata_root, product, debug, extras=None): + self._update_mozinfo(metadata_root) + self.update(mozinfo.info) + self["product"] = product + if debug is not None: + self["debug"] = debug + elif "debug" not in self: + # Default to release + self["debug"] = False + if extras is not None: + self.update(extras) + + def _update_mozinfo(self, metadata_root): + """Add extra build information from a mozinfo.json file in a parent + directory""" + path = metadata_root + dirs = set() + while path != os.path.expanduser('~'): + if path in dirs: + break + dirs.add(str(path)) + path = os.path.split(path)[0] + + mozinfo.find_and_update_from_json(*dirs) + + +class B2GRunInfo(RunInfo): + def __init__(self, *args, **kwargs): + RunInfo.__init__(self, *args, **kwargs) + self["os"] = "b2g" + + +class Test(object): + result_cls = None + subtest_result_cls = None + test_type = None + + def __init__(self, tests_root, url, inherit_metadata, test_metadata, + timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): + self.tests_root = tests_root + self.url = url + self._inherit_metadata = inherit_metadata + self._test_metadata = test_metadata + self.timeout = timeout + self.path = path + self.environment = {"protocol": protocol, "prefs": self.prefs} + + def __eq__(self, other): + return self.id == other.id + + @classmethod + def from_manifest(cls, manifest_item, inherit_metadata, test_metadata): + timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT + protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http" + return cls(manifest_item.source_file.tests_root, + manifest_item.url, + inherit_metadata, + test_metadata, + timeout=timeout, + path=manifest_item.path, + protocol=protocol) + + @property + def id(self): + return self.url + + @property + def keys(self): + return tuple() + + @property + def abs_path(self): + return os.path.join(self.tests_root, self.path) + + def _get_metadata(self, subtest=None): + if self._test_metadata is not None and subtest is not None: + return self._test_metadata.get_subtest(subtest) + else: + return self._test_metadata + + def itermeta(self, subtest=None): + for metadata in self._inherit_metadata: + yield metadata + + if self._test_metadata is not None: + yield self._get_metadata() + if subtest is not None: + subtest_meta = self._get_metadata(subtest) + if subtest_meta is not None: + yield subtest_meta + + def disabled(self, subtest=None): + for meta in self.itermeta(subtest): + disabled = meta.disabled + if disabled is not None: + return disabled + return None + + @property + def restart_after(self): + for meta in self.itermeta(None): + restart_after = meta.restart_after + if restart_after is not None: + return True + return False + + @property + def tags(self): + tags = set() + for meta in self.itermeta(): + meta_tags = meta.tags + if atom_reset in meta_tags: + tags = meta_tags.copy() + tags.remove(atom_reset) + else: + tags |= meta_tags + + tags.add("dir:%s" % self.id.lstrip("/").split("/")[0]) + + return tags + + @property + def prefs(self): + prefs = {} + for meta in self.itermeta(): + meta_prefs = meta.prefs + if atom_reset in prefs: + prefs = meta_prefs.copy() + del prefs[atom_reset] + else: + prefs.update(meta_prefs) + return prefs + + def expected(self, subtest=None): + if subtest is None: + default = self.result_cls.default_expected + else: + default = self.subtest_result_cls.default_expected + + metadata = self._get_metadata(subtest) + if metadata is None: + return default + + try: + return metadata.get("expected") + except KeyError: + return default + + def __repr__(self): + return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id) + + +class TestharnessTest(Test): + result_cls = TestharnessResult + subtest_result_cls = TestharnessSubtestResult + test_type = "testharness" + + @property + def id(self): + return self.url + + +class ManualTest(Test): + test_type = "manual" + + @property + def id(self): + return self.url + + +class ReftestTest(Test): + result_cls = ReftestResult + test_type = "reftest" + + def __init__(self, tests_root, url, inherit_metadata, test_metadata, references, + timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None, + dpi=None, protocol="http"): + Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout, + path, protocol) + + for _, ref_type in references: + if ref_type not in ("==", "!="): + raise ValueError + + self.references = references + self.viewport_size = viewport_size + self.dpi = dpi + + @classmethod + def from_manifest(cls, + manifest_test, + inherit_metadata, + test_metadata, + nodes=None, + references_seen=None): + + timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT + + if nodes is None: + nodes = {} + if references_seen is None: + references_seen = set() + + url = manifest_test.url + + node = cls(manifest_test.source_file.tests_root, + manifest_test.url, + inherit_metadata, + test_metadata, + [], + timeout=timeout, + path=manifest_test.path, + viewport_size=manifest_test.viewport_size, + dpi=manifest_test.dpi, + protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http") + + nodes[url] = node + + for ref_url, ref_type in manifest_test.references: + comparison_key = (ref_type,) + tuple(sorted([url, ref_url])) + if ref_url in nodes: + manifest_node = ref_url + if comparison_key in references_seen: + # We have reached a cycle so stop here + # Note that just seeing a node for the second time is not + # enough to detect a cycle because + # A != B != C != A must include C != A + # but A == B == A should not include the redundant B == A. + continue + + references_seen.add(comparison_key) + + manifest_node = manifest_test.manifest.get_reference(ref_url) + if manifest_node: + reference = ReftestTest.from_manifest(manifest_node, + [], + None, + nodes, + references_seen) + else: + reference = ReftestTest(manifest_test.source_file.tests_root, + ref_url, + [], + None, + []) + + node.references.append((reference, ref_type)) + + return node + + @property + def id(self): + return self.url + + @property + def keys(self): + return ("reftype", "refurl") + + +class WdspecTest(Test): + result_cls = WdspecResult + subtest_result_cls = WdspecSubtestResult + test_type = "wdspec" + + +manifest_test_cls = {"reftest": ReftestTest, + "testharness": TestharnessTest, + "manual": ManualTest, + "wdspec": WdspecTest} + + +def from_manifest(manifest_test, inherit_metadata, test_metadata): + test_cls = manifest_test_cls[manifest_test.item_type] + return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata) |