summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/harness/wptrunner/wpttest.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/web-platform/harness/wptrunner/wpttest.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/web-platform/harness/wptrunner/wpttest.py')
-rw-r--r--testing/web-platform/harness/wptrunner/wpttest.py351
1 files changed, 351 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/wpttest.py b/testing/web-platform/harness/wptrunner/wpttest.py
new file mode 100644
index 000000000..177e9208a
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -0,0 +1,351 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DEFAULT_TIMEOUT = 10 # seconds
+LONG_TIMEOUT = 60 # seconds
+
+import os
+
+import mozinfo
+
+from wptmanifest.parser import atoms
+
+atom_reset = atoms["Reset"]
+enabled_tests = set(["testharness", "reftest", "wdspec"])
+
+
+class Result(object):
+ def __init__(self, status, message, expected=None, extra=None):
+ if status not in self.statuses:
+ raise ValueError("Unrecognised status %s" % status)
+ self.status = status
+ self.message = message
+ self.expected = expected
+ self.extra = extra
+
+ def __repr__(self):
+ return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
+
+
+class SubtestResult(object):
+ def __init__(self, name, status, message, stack=None, expected=None):
+ self.name = name
+ if status not in self.statuses:
+ raise ValueError("Unrecognised status %s" % status)
+ self.status = status
+ self.message = message
+ self.stack = stack
+ self.expected = expected
+
+ def __repr__(self):
+ return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
+
+
+class TestharnessResult(Result):
+ default_expected = "OK"
+ statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
+
+
+class TestharnessSubtestResult(SubtestResult):
+ default_expected = "PASS"
+ statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
+
+
+class ReftestResult(Result):
+ default_expected = "PASS"
+ statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
+
+
+class WdspecResult(Result):
+ default_expected = "OK"
+ statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
+
+
+class WdspecSubtestResult(SubtestResult):
+ default_expected = "PASS"
+ statuses = set(["PASS", "FAIL", "ERROR"])
+
+
+def get_run_info(metadata_root, product, **kwargs):
+ if product == "b2g":
+ return B2GRunInfo(metadata_root, product, **kwargs)
+ else:
+ return RunInfo(metadata_root, product, **kwargs)
+
+
+class RunInfo(dict):
+ def __init__(self, metadata_root, product, debug, extras=None):
+ self._update_mozinfo(metadata_root)
+ self.update(mozinfo.info)
+ self["product"] = product
+ if debug is not None:
+ self["debug"] = debug
+ elif "debug" not in self:
+ # Default to release
+ self["debug"] = False
+ if extras is not None:
+ self.update(extras)
+
+ def _update_mozinfo(self, metadata_root):
+ """Add extra build information from a mozinfo.json file in a parent
+ directory"""
+ path = metadata_root
+ dirs = set()
+ while path != os.path.expanduser('~'):
+ if path in dirs:
+ break
+ dirs.add(str(path))
+ path = os.path.split(path)[0]
+
+ mozinfo.find_and_update_from_json(*dirs)
+
+
+class B2GRunInfo(RunInfo):
+ def __init__(self, *args, **kwargs):
+ RunInfo.__init__(self, *args, **kwargs)
+ self["os"] = "b2g"
+
+
+class Test(object):
+ result_cls = None
+ subtest_result_cls = None
+ test_type = None
+
+ def __init__(self, tests_root, url, inherit_metadata, test_metadata,
+ timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
+ self.tests_root = tests_root
+ self.url = url
+ self._inherit_metadata = inherit_metadata
+ self._test_metadata = test_metadata
+ self.timeout = timeout
+ self.path = path
+ self.environment = {"protocol": protocol, "prefs": self.prefs}
+
+ def __eq__(self, other):
+ return self.id == other.id
+
+ @classmethod
+ def from_manifest(cls, manifest_item, inherit_metadata, test_metadata):
+ timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
+ protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http"
+ return cls(manifest_item.source_file.tests_root,
+ manifest_item.url,
+ inherit_metadata,
+ test_metadata,
+ timeout=timeout,
+ path=manifest_item.path,
+ protocol=protocol)
+
+ @property
+ def id(self):
+ return self.url
+
+ @property
+ def keys(self):
+ return tuple()
+
+ @property
+ def abs_path(self):
+ return os.path.join(self.tests_root, self.path)
+
+ def _get_metadata(self, subtest=None):
+ if self._test_metadata is not None and subtest is not None:
+ return self._test_metadata.get_subtest(subtest)
+ else:
+ return self._test_metadata
+
+ def itermeta(self, subtest=None):
+ for metadata in self._inherit_metadata:
+ yield metadata
+
+ if self._test_metadata is not None:
+ yield self._get_metadata()
+ if subtest is not None:
+ subtest_meta = self._get_metadata(subtest)
+ if subtest_meta is not None:
+ yield subtest_meta
+
+ def disabled(self, subtest=None):
+ for meta in self.itermeta(subtest):
+ disabled = meta.disabled
+ if disabled is not None:
+ return disabled
+ return None
+
+ @property
+ def restart_after(self):
+ for meta in self.itermeta(None):
+ restart_after = meta.restart_after
+ if restart_after is not None:
+ return True
+ return False
+
+ @property
+ def tags(self):
+ tags = set()
+ for meta in self.itermeta():
+ meta_tags = meta.tags
+ if atom_reset in meta_tags:
+ tags = meta_tags.copy()
+ tags.remove(atom_reset)
+ else:
+ tags |= meta_tags
+
+ tags.add("dir:%s" % self.id.lstrip("/").split("/")[0])
+
+ return tags
+
+ @property
+ def prefs(self):
+ prefs = {}
+ for meta in self.itermeta():
+ meta_prefs = meta.prefs
+ if atom_reset in prefs:
+ prefs = meta_prefs.copy()
+ del prefs[atom_reset]
+ else:
+ prefs.update(meta_prefs)
+ return prefs
+
+ def expected(self, subtest=None):
+ if subtest is None:
+ default = self.result_cls.default_expected
+ else:
+ default = self.subtest_result_cls.default_expected
+
+ metadata = self._get_metadata(subtest)
+ if metadata is None:
+ return default
+
+ try:
+ return metadata.get("expected")
+ except KeyError:
+ return default
+
+ def __repr__(self):
+ return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
+
+
+class TestharnessTest(Test):
+ result_cls = TestharnessResult
+ subtest_result_cls = TestharnessSubtestResult
+ test_type = "testharness"
+
+ @property
+ def id(self):
+ return self.url
+
+
+class ManualTest(Test):
+ test_type = "manual"
+
+ @property
+ def id(self):
+ return self.url
+
+
+class ReftestTest(Test):
+ result_cls = ReftestResult
+ test_type = "reftest"
+
+ def __init__(self, tests_root, url, inherit_metadata, test_metadata, references,
+ timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
+ dpi=None, protocol="http"):
+ Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
+ path, protocol)
+
+ for _, ref_type in references:
+ if ref_type not in ("==", "!="):
+ raise ValueError
+
+ self.references = references
+ self.viewport_size = viewport_size
+ self.dpi = dpi
+
+ @classmethod
+ def from_manifest(cls,
+ manifest_test,
+ inherit_metadata,
+ test_metadata,
+ nodes=None,
+ references_seen=None):
+
+ timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT
+
+ if nodes is None:
+ nodes = {}
+ if references_seen is None:
+ references_seen = set()
+
+ url = manifest_test.url
+
+ node = cls(manifest_test.source_file.tests_root,
+ manifest_test.url,
+ inherit_metadata,
+ test_metadata,
+ [],
+ timeout=timeout,
+ path=manifest_test.path,
+ viewport_size=manifest_test.viewport_size,
+ dpi=manifest_test.dpi,
+ protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
+
+ nodes[url] = node
+
+ for ref_url, ref_type in manifest_test.references:
+ comparison_key = (ref_type,) + tuple(sorted([url, ref_url]))
+ if ref_url in nodes:
+ manifest_node = ref_url
+ if comparison_key in references_seen:
+ # We have reached a cycle so stop here
+ # Note that just seeing a node for the second time is not
+ # enough to detect a cycle because
+ # A != B != C != A must include C != A
+ # but A == B == A should not include the redundant B == A.
+ continue
+
+ references_seen.add(comparison_key)
+
+ manifest_node = manifest_test.manifest.get_reference(ref_url)
+ if manifest_node:
+ reference = ReftestTest.from_manifest(manifest_node,
+ [],
+ None,
+ nodes,
+ references_seen)
+ else:
+ reference = ReftestTest(manifest_test.source_file.tests_root,
+ ref_url,
+ [],
+ None,
+ [])
+
+ node.references.append((reference, ref_type))
+
+ return node
+
+ @property
+ def id(self):
+ return self.url
+
+ @property
+ def keys(self):
+ return ("reftype", "refurl")
+
+
+class WdspecTest(Test):
+ result_cls = WdspecResult
+ subtest_result_cls = WdspecSubtestResult
+ test_type = "wdspec"
+
+
+manifest_test_cls = {"reftest": ReftestTest,
+ "testharness": TestharnessTest,
+ "manual": ManualTest,
+ "wdspec": WdspecTest}
+
+
+def from_manifest(manifest_test, inherit_metadata, test_metadata):
+ test_cls = manifest_test_cls[manifest_test.item_type]
+ return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata)