diff options
Diffstat (limited to 'testing/web-platform/tests/tools/manifest')
13 files changed, 1711 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/manifest/__init__.py b/testing/web-platform/tests/tools/manifest/__init__.py new file mode 100644 index 000000000..7ecb04be9 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/__init__.py @@ -0,0 +1,5 @@ +from . import item +from . import manifest +from . import sourcefile +from . import tree +from . import update diff --git a/testing/web-platform/tests/tools/manifest/item.py b/testing/web-platform/tests/tools/manifest/item.py new file mode 100644 index 000000000..76c91697f --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/item.py @@ -0,0 +1,191 @@ +import os +from six.moves.urllib.parse import urljoin +from abc import ABCMeta, abstractmethod, abstractproperty + +from .utils import from_os_path, to_os_path + +item_types = ["testharness", "reftest", "manual", "stub", "wdspec"] + + +def get_source_file(source_files, tests_root, manifest, path): + def make_new(): + from .sourcefile import SourceFile + + return SourceFile(tests_root, path, manifest.url_base) + + if source_files is None: + return make_new() + + if path not in source_files: + source_files[path] = make_new() + + return source_files[path] + + +class ManifestItem(object): + __metaclass__ = ABCMeta + + item_type = None + + def __init__(self, source_file, manifest=None): + self.manifest = manifest + self.source_file = source_file + + @abstractproperty + def id(self): + """The test's id (usually its url)""" + pass + + @property + def path(self): + """The test path relative to the test_root""" + return self.source_file.rel_path + + @property + def https(self): + return "https" in self.source_file.meta_flags + + def key(self): + """A unique identifier for the test""" + return (self.item_type, self.id) + + def meta_key(self): + """Extra metadata that doesn't form part of the test identity, but for + which changes mean regenerating the manifest (e.g. the test timeout.""" + return () + + def __eq__(self, other): + if not hasattr(other, "key"): + return False + return self.key() == other.key() + + def __hash__(self): + return hash(self.key() + self.meta_key()) + + def __repr__(self): + return "<%s.%s id=%s, path=%s>" % (self.__module__, self.__class__.__name__, self.id, self.path) + + def to_json(self): + return {"path": from_os_path(self.path)} + + @classmethod + def from_json(self, manifest, tests_root, obj, source_files=None): + raise NotImplementedError + + +class URLManifestItem(ManifestItem): + def __init__(self, source_file, url, url_base="/", manifest=None): + ManifestItem.__init__(self, source_file, manifest=manifest) + self._url = url + self.url_base = url_base + + @property + def id(self): + return self.url + + @property + def url(self): + return urljoin(self.url_base, self._url) + + def to_json(self): + rv = ManifestItem.to_json(self) + rv["url"] = self._url + return rv + + @classmethod + def from_json(cls, manifest, tests_root, obj, source_files=None): + source_file = get_source_file(source_files, tests_root, manifest, + to_os_path(obj["path"])) + return cls(source_file, + obj["url"], + url_base=manifest.url_base, + manifest=manifest) + + +class TestharnessTest(URLManifestItem): + item_type = "testharness" + + def __init__(self, source_file, url, url_base="/", timeout=None, manifest=None): + URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest) + self.timeout = timeout + + def meta_key(self): + return (self.timeout,) + + def to_json(self): + rv = URLManifestItem.to_json(self) + if self.timeout is not None: + rv["timeout"] = self.timeout + return rv + + @classmethod + def from_json(cls, manifest, tests_root, obj, source_files=None): + source_file = get_source_file(source_files, tests_root, manifest, + to_os_path(obj["path"])) + return cls(source_file, + obj["url"], + url_base=manifest.url_base, + timeout=obj.get("timeout"), + manifest=manifest) + + +class RefTest(URLManifestItem): + item_type = "reftest" + + def __init__(self, source_file, url, references, url_base="/", timeout=None, + viewport_size=None, dpi=None, manifest=None): + URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest) + for _, ref_type in references: + if ref_type not in ["==", "!="]: + raise ValueError("Unrecognised ref_type %s" % ref_type) + self.references = tuple(references) + self.timeout = timeout + self.viewport_size = viewport_size + self.dpi = dpi + + @property + def is_reference(self): + return self.source_file.name_is_reference + + def meta_key(self): + return (self.timeout, self.viewport_size, self.dpi) + + def to_json(self): + rv = URLManifestItem.to_json(self) + rv["references"] = self.references + if self.timeout is not None: + rv["timeout"] = self.timeout + if self.viewport_size is not None: + rv["viewport_size"] = self.viewport_size + if self.dpi is not None: + rv["dpi"] = self.dpi + return rv + + @classmethod + def from_json(cls, manifest, tests_root, obj, source_files=None): + source_file = get_source_file(source_files, tests_root, manifest, + to_os_path(obj["path"])) + return cls(source_file, + obj["url"], + obj["references"], + url_base=manifest.url_base, + timeout=obj.get("timeout"), + viewport_size=obj.get("viewport_size"), + dpi=obj.get("dpi"), + manifest=manifest) + + +class ManualTest(URLManifestItem): + item_type = "manual" + + +class Stub(URLManifestItem): + item_type = "stub" + + +class WebdriverSpecTest(URLManifestItem): + item_type = "wdspec" + + def __init__(self, source_file, url, url_base="/", timeout=None, manifest=None): + URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest) + self.timeout = timeout diff --git a/testing/web-platform/tests/tools/manifest/log.py b/testing/web-platform/tests/tools/manifest/log.py new file mode 100644 index 000000000..affb7d306 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/log.py @@ -0,0 +1,8 @@ +import logging + +logger = logging.getLogger("manifest") +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.DEBUG) + +def get_logger(): + return logger diff --git a/testing/web-platform/tests/tools/manifest/manifest.py b/testing/web-platform/tests/tools/manifest/manifest.py new file mode 100644 index 000000000..80fe70f95 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/manifest.py @@ -0,0 +1,418 @@ +import json +import os +from collections import defaultdict, OrderedDict +from six import iteritems + +from .item import item_types, ManualTest, WebdriverSpecTest, Stub, RefTest, TestharnessTest +from .log import get_logger +from .sourcefile import SourceFile +from .utils import from_os_path, to_os_path + + +CURRENT_VERSION = 3 + + +class ManifestError(Exception): + pass + + +class ManifestVersionMismatch(ManifestError): + pass + +class Manifest(object): + def __init__(self, git_rev=None, url_base="/"): + # Dict of item_type: {path: set(manifest_items)} + self._data = dict((item_type, defaultdict(set)) + for item_type in item_types) + self.rev = git_rev + self.url_base = url_base + self.local_changes = LocalChanges(self) + # reftest nodes arranged as {path: set(manifest_items)} + self.reftest_nodes = defaultdict(set) + self.reftest_nodes_by_url = {} + + def _included_items(self, include_types=None): + if include_types is None: + include_types = item_types + + for item_type in include_types: + paths = self._data[item_type].copy() + for local_types, local_paths in self.local_changes.itertypes(item_type): + for path, items in iteritems(local_paths): + paths[path] = items + for path in self.local_changes.iterdeleted(): + if path in paths: + del paths[path] + if item_type == "reftest": + for path, items in self.local_changes.iterdeletedreftests(): + paths[path] -= items + if len(paths[path]) == 0: + del paths[path] + + yield item_type, paths + + def contains_path(self, path): + return any(path in paths for _, paths in self._included_items()) + + def add(self, item): + if item is None: + return + + if isinstance(item, RefTest): + self.reftest_nodes[item.path].add(item) + self.reftest_nodes_by_url[item.url] = item + else: + self._add(item) + + item.manifest = self + + def _add(self, item): + self._data[item.item_type][item.path].add(item) + + def extend(self, items): + for item in items: + self.add(item) + + def remove_path(self, path): + for item_type in item_types: + if path in self._data[item_type]: + del self._data[item_type][path] + + def itertypes(self, *types): + if not types: + types = None + for item_type, items in self._included_items(types): + for item in sorted(iteritems(items)): + yield item + + def __iter__(self): + for item in self.itertypes(): + yield item + + def __getitem__(self, path): + for _, paths in self._included_items(): + if path in paths: + return paths[path] + raise KeyError + + def get_reference(self, url): + if url in self.local_changes.reftest_nodes_by_url: + return self.local_changes.reftest_nodes_by_url[url] + + if url in self.reftest_nodes_by_url: + return self.reftest_nodes_by_url[url] + + return None + + def _committed_with_path(self, rel_path): + rv = set() + + for paths_items in self._data.itervalues(): + rv |= paths_items.get(rel_path, set()) + + if rel_path in self.reftest_nodes: + rv |= self.reftest_nodes[rel_path] + + return rv + + def _committed_paths(self): + rv = set() + for paths_items in self._data.itervalues(): + rv |= set(paths_items.keys()) + return rv + + def update(self, + tests_root, + url_base, + new_rev, + committed_changes=None, + local_changes=None, + remove_missing_local=False): + + if local_changes is None: + local_changes = {} + + if committed_changes is not None: + for rel_path, status in committed_changes: + self.remove_path(rel_path) + if status == "modified": + use_committed = rel_path in local_changes + source_file = SourceFile(tests_root, + rel_path, + url_base, + use_committed=use_committed) + self.extend(source_file.manifest_items()) + + self.local_changes = LocalChanges(self) + + local_paths = set() + for rel_path, status in iteritems(local_changes): + local_paths.add(rel_path) + + if status == "modified": + existing_items = self._committed_with_path(rel_path) + source_file = SourceFile(tests_root, + rel_path, + url_base, + use_committed=False) + local_items = set(source_file.manifest_items()) + + updated_items = local_items - existing_items + self.local_changes.extend(updated_items) + else: + self.local_changes.add_deleted(rel_path) + + if remove_missing_local: + for path in self._committed_paths() - local_paths: + self.local_changes.add_deleted(path) + + self.update_reftests() + + if new_rev is not None: + self.rev = new_rev + self.url_base = url_base + + def update_reftests(self): + default_reftests = self.compute_reftests(self.reftest_nodes) + all_reftest_nodes = self.reftest_nodes.copy() + all_reftest_nodes.update(self.local_changes.reftest_nodes) + + for item in self.local_changes.iterdeleted(): + if item in all_reftest_nodes: + del all_reftest_nodes[item] + + modified_reftests = self.compute_reftests(all_reftest_nodes) + + added_reftests = modified_reftests - default_reftests + # The interesting case here is not when the file is deleted, + # but when a reftest like A == B is changed to the form + # C == A == B, so that A still exists but is now a ref rather than + # a test. + removed_reftests = default_reftests - modified_reftests + + dests = [(default_reftests, self._data["reftest"]), + (added_reftests, self.local_changes._data["reftest"]), + (removed_reftests, self.local_changes._deleted_reftests)] + + #TODO: Warn if there exist unreachable reftest nodes + for source, target in dests: + for item in source: + target[item.path].add(item) + + def compute_reftests(self, reftest_nodes): + """Given a set of reftest_nodes, return a set of all the nodes that are top-level + tests i.e. don't have any incoming reference links.""" + + reftests = set() + + has_inbound = set() + for path, items in iteritems(reftest_nodes): + for item in items: + for ref_url, ref_type in item.references: + has_inbound.add(ref_url) + + for path, items in iteritems(reftest_nodes): + for item in items: + if item.url in has_inbound: + continue + reftests.add(item) + + return reftests + + def to_json(self): + out_items = { + item_type: sorted( + test.to_json() + for _, tests in iteritems(items) + for test in tests + ) + for item_type, items in iteritems(self._data) + } + + reftest_nodes = OrderedDict() + for key, value in sorted(iteritems(self.reftest_nodes)): + reftest_nodes[from_os_path(key)] = [v.to_json() for v in value] + + rv = {"url_base": self.url_base, + "rev": self.rev, + "local_changes": self.local_changes.to_json(), + "items": out_items, + "reftest_nodes": reftest_nodes, + "version": CURRENT_VERSION} + return rv + + @classmethod + def from_json(cls, tests_root, obj): + version = obj.get("version") + if version != CURRENT_VERSION: + raise ManifestVersionMismatch + + self = cls(git_rev=obj["rev"], + url_base=obj.get("url_base", "/")) + if not hasattr(obj, "items"): + raise ManifestError + + item_classes = {"testharness": TestharnessTest, + "reftest": RefTest, + "manual": ManualTest, + "stub": Stub, + "wdspec": WebdriverSpecTest} + + source_files = {} + + for k, values in iteritems(obj["items"]): + if k not in item_types: + raise ManifestError + for v in values: + manifest_item = item_classes[k].from_json(self, tests_root, v, + source_files=source_files) + self._add(manifest_item) + + for path, values in iteritems(obj["reftest_nodes"]): + path = to_os_path(path) + for v in values: + item = RefTest.from_json(self, tests_root, v, + source_files=source_files) + self.reftest_nodes[path].add(item) + self.reftest_nodes_by_url[v["url"]] = item + + self.local_changes = LocalChanges.from_json(self, + tests_root, + obj["local_changes"], + source_files=source_files) + self.update_reftests() + return self + + +class LocalChanges(object): + def __init__(self, manifest): + self.manifest = manifest + self._data = dict((item_type, defaultdict(set)) for item_type in item_types) + self._deleted = set() + self.reftest_nodes = defaultdict(set) + self.reftest_nodes_by_url = {} + self._deleted_reftests = defaultdict(set) + + def add(self, item): + if item is None: + return + + if isinstance(item, RefTest): + self.reftest_nodes[item.path].add(item) + self.reftest_nodes_by_url[item.url] = item + else: + self._add(item) + + item.manifest = self.manifest + + def _add(self, item): + self._data[item.item_type][item.path].add(item) + + def extend(self, items): + for item in items: + self.add(item) + + def add_deleted(self, path): + self._deleted.add(path) + + def is_deleted(self, path): + return path in self._deleted + + def itertypes(self, *types): + for item_type in types: + yield item_type, self._data[item_type] + + def iterdeleted(self): + for item in self._deleted: + yield item + + def iterdeletedreftests(self): + for item in iteritems(self._deleted_reftests): + yield item + + def __getitem__(self, item_type): + return self._data[item_type] + + def to_json(self): + reftest_nodes = {from_os_path(key): [v.to_json() for v in value] + for key, value in iteritems(self.reftest_nodes)} + + deleted_reftests = {from_os_path(key): [v.to_json() for v in value] + for key, value in iteritems(self._deleted_reftests)} + + rv = {"items": defaultdict(dict), + "reftest_nodes": reftest_nodes, + "deleted": [from_os_path(path) for path in self._deleted], + "deleted_reftests": deleted_reftests} + + for test_type, paths in iteritems(self._data): + for path, tests in iteritems(paths): + path = from_os_path(path) + rv["items"][test_type][path] = [test.to_json() for test in tests] + + return rv + + @classmethod + def from_json(cls, manifest, tests_root, obj, source_files=None): + self = cls(manifest) + if not hasattr(obj, "items"): + raise ManifestError + + item_classes = {"testharness": TestharnessTest, + "reftest": RefTest, + "manual": ManualTest, + "stub": Stub, + "wdspec": WebdriverSpecTest} + + for test_type, paths in iteritems(obj["items"]): + for path, tests in iteritems(paths): + for test in tests: + manifest_item = item_classes[test_type].from_json(manifest, + tests_root, + test, + source_files=source_files) + self.add(manifest_item) + + for path, values in iteritems(obj["reftest_nodes"]): + path = to_os_path(path) + for v in values: + item = RefTest.from_json(self.manifest, tests_root, v, + source_files=source_files) + self.reftest_nodes[path].add(item) + self.reftest_nodes_by_url[item.url] = item + + for item in obj["deleted"]: + self.add_deleted(to_os_path(item)) + + for path, values in iteritems(obj.get("deleted_reftests", {})): + path = to_os_path(path) + for v in values: + item = RefTest.from_json(self.manifest, tests_root, v, + source_files=source_files) + self._deleted_reftests[path].add(item) + + return self + +def load(tests_root, manifest): + logger = get_logger() + + # "manifest" is a path or file-like object. + if isinstance(manifest, basestring): + if os.path.exists(manifest): + logger.debug("Opening manifest at %s" % manifest) + else: + logger.debug("Creating new manifest at %s" % manifest) + try: + with open(manifest) as f: + rv = Manifest.from_json(tests_root, json.load(f)) + except IOError: + rv = Manifest(None) + return rv + + return Manifest.from_json(tests_root, json.load(manifest)) + + +def write(manifest, manifest_path): + with open(manifest_path, "wb") as f: + json.dump(manifest.to_json(), f, sort_keys=True, indent=2, separators=(',', ': ')) + f.write("\n") diff --git a/testing/web-platform/tests/tools/manifest/sourcefile.py b/testing/web-platform/tests/tools/manifest/sourcefile.py new file mode 100644 index 000000000..44a462707 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/sourcefile.py @@ -0,0 +1,366 @@ +import imp +import os +import re +from six.moves.urllib.parse import urljoin +from fnmatch import fnmatch +try: + from xml.etree import cElementTree as ElementTree +except ImportError: + from xml.etree import ElementTree + +import html5lib + +from . import vcs +from .item import Stub, ManualTest, WebdriverSpecTest, RefTest, TestharnessTest +from .utils import rel_path_to_url, is_blacklisted, ContextManagerBytesIO, cached_property + +wd_pattern = "*.py" +meta_re = re.compile("//\s*<meta>\s*(\w*)=(.*)$") + +def replace_end(s, old, new): + """ + Given a string `s` that ends with `old`, replace that occurrence of `old` + with `new`. + """ + assert s.endswith(old) + return s[:-len(old)] + new + + +class SourceFile(object): + parsers = {"html":lambda x:html5lib.parse(x, treebuilder="etree"), + "xhtml":ElementTree.parse, + "svg":ElementTree.parse} + + def __init__(self, tests_root, rel_path, url_base, use_committed=False, + contents=None): + """Object representing a file in a source tree. + + :param tests_root: Path to the root of the source tree + :param rel_path: File path relative to tests_root + :param url_base: Base URL used when converting file paths to urls + :param use_committed: Work with the last committed version of the file + rather than the on-disk version. + :param contents: Byte array of the contents of the file or ``None``. + """ + + assert not (use_committed and contents is not None) + + self.tests_root = tests_root + self.rel_path = rel_path + self.url_base = url_base + self.use_committed = use_committed + self.contents = contents + + self.url = rel_path_to_url(rel_path, url_base) + self.path = os.path.join(tests_root, rel_path) + + self.dir_path, self.filename = os.path.split(self.path) + self.name, self.ext = os.path.splitext(self.filename) + + self.type_flag = None + if "-" in self.name: + self.type_flag = self.name.rsplit("-", 1)[1].split(".")[0] + + self.meta_flags = self.name.split(".")[1:] + + def __getstate__(self): + # Remove computed properties if we pickle this class + rv = self.__dict__.copy() + + if "__cached_properties__" in rv: + cached_properties = rv["__cached_properties__"] + for key in rv.keys(): + if key in cached_properties: + del rv[key] + del rv["__cached_properties__"] + return rv + + def name_prefix(self, prefix): + """Check if the filename starts with a given prefix + + :param prefix: The prefix to check""" + return self.name.startswith(prefix) + + def is_dir(self): + """Return whether this file represents a directory.""" + if self.contents is not None: + return False + + return os.path.isdir(self.rel_path) + + def open(self): + """ + Return either + * the contents specified in the constructor, if any; + * the contents of the file when last committed, if use_committed is true; or + * a File object opened for reading the file contents. + """ + + if self.contents is not None: + file_obj = ContextManagerBytesIO(self.contents) + elif self.use_committed: + git = vcs.get_git_func(os.path.dirname(__file__)) + blob = git("show", "HEAD:%s" % self.rel_path) + file_obj = ContextManagerBytesIO(blob) + else: + file_obj = open(self.path, 'rb') + return file_obj + + @property + def name_is_non_test(self): + """Check if the file name matches the conditions for the file to + be a non-test file""" + return (self.is_dir() or + self.name_prefix("MANIFEST") or + self.filename.startswith(".") or + is_blacklisted(self.url)) + + @property + def name_is_stub(self): + """Check if the file name matches the conditions for the file to + be a stub file""" + return self.name_prefix("stub-") + + @property + def name_is_manual(self): + """Check if the file name matches the conditions for the file to + be a manual test file""" + return self.type_flag == "manual" + + @property + def name_is_multi_global(self): + """Check if the file name matches the conditions for the file to + be a multi-global js test file""" + return "any" in self.meta_flags and self.ext == ".js" + + @property + def name_is_worker(self): + """Check if the file name matches the conditions for the file to + be a worker js test file""" + return "worker" in self.meta_flags and self.ext == ".js" + + @property + def name_is_webdriver(self): + """Check if the file name matches the conditions for the file to + be a webdriver spec test file""" + # wdspec tests are in subdirectories of /webdriver excluding __init__.py + # files. + rel_dir_tree = self.rel_path.split(os.path.sep) + return (rel_dir_tree[0] == "webdriver" and + len(rel_dir_tree) > 1 and + self.filename != "__init__.py" and + fnmatch(self.filename, wd_pattern)) + + @property + def name_is_reference(self): + """Check if the file name matches the conditions for the file to + be a reference file (not a reftest)""" + return self.type_flag in ("ref", "notref") + + @property + def markup_type(self): + """Return the type of markup contained in a file, based on its extension, + or None if it doesn't contain markup""" + ext = self.ext + + if not ext: + return None + if ext[0] == ".": + ext = ext[1:] + if ext in ["html", "htm"]: + return "html" + if ext in ["xhtml", "xht", "xml"]: + return "xhtml" + if ext == "svg": + return "svg" + return None + + @cached_property + def root(self): + """Return an ElementTree Element for the root node of the file if it contains + markup, or None if it does not""" + if not self.markup_type: + return None + + parser = self.parsers[self.markup_type] + + with self.open() as f: + try: + tree = parser(f) + except Exception: + return None + + if hasattr(tree, "getroot"): + root = tree.getroot() + else: + root = tree + + return root + + @cached_property + def timeout_nodes(self): + """List of ElementTree Elements corresponding to nodes in a test that + specify timeouts""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='timeout']") + + @cached_property + def timeout(self): + """The timeout of a test or reference file. "long" if the file has an extended timeout + or None otherwise""" + if self.name_is_worker: + with self.open() as f: + for line in f: + m = meta_re.match(line) + if m and m.groups()[0] == "timeout": + if m.groups()[1].lower() == "long": + return "long" + return + + if not self.root: + return + + if self.timeout_nodes: + timeout_str = self.timeout_nodes[0].attrib.get("content", None) + if timeout_str and timeout_str.lower() == "long": + return timeout_str + + @cached_property + def viewport_nodes(self): + """List of ElementTree Elements corresponding to nodes in a test that + specify viewport sizes""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='viewport-size']") + + @cached_property + def viewport_size(self): + """The viewport size of a test or reference file""" + if not self.root: + return None + + if not self.viewport_nodes: + return None + + return self.viewport_nodes[0].attrib.get("content", None) + + @cached_property + def dpi_nodes(self): + """List of ElementTree Elements corresponding to nodes in a test that + specify device pixel ratios""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='device-pixel-ratio']") + + @cached_property + def dpi(self): + """The device pixel ratio of a test or reference file""" + if not self.root: + return None + + if not self.dpi_nodes: + return None + + return self.dpi_nodes[0].attrib.get("content", None) + + @cached_property + def testharness_nodes(self): + """List of ElementTree Elements corresponding to nodes representing a + testharness.js script""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testharness.js']") + + @cached_property + def content_is_testharness(self): + """Boolean indicating whether the file content represents a + testharness.js test""" + if not self.root: + return None + return bool(self.testharness_nodes) + + @cached_property + def variant_nodes(self): + """List of ElementTree Elements corresponding to nodes representing a + test variant""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='variant']") + + @cached_property + def test_variants(self): + rv = [] + for element in self.variant_nodes: + if "content" in element.attrib: + variant = element.attrib["content"] + assert variant == "" or variant[0] in ["#", "?"] + rv.append(variant) + + if not rv: + rv = [""] + + return rv + + @cached_property + def reftest_nodes(self): + """List of ElementTree Elements corresponding to nodes representing a + to a reftest <link>""" + if not self.root: + return [] + + match_links = self.root.findall(".//{http://www.w3.org/1999/xhtml}link[@rel='match']") + mismatch_links = self.root.findall(".//{http://www.w3.org/1999/xhtml}link[@rel='mismatch']") + return match_links + mismatch_links + + @cached_property + def references(self): + """List of (ref_url, relation) tuples for any reftest references specified in + the file""" + rv = [] + rel_map = {"match": "==", "mismatch": "!="} + for item in self.reftest_nodes: + if "href" in item.attrib: + ref_url = urljoin(self.url, item.attrib["href"]) + ref_type = rel_map[item.attrib["rel"]] + rv.append((ref_url, ref_type)) + return rv + + @cached_property + def content_is_ref_node(self): + """Boolean indicating whether the file is a non-leaf node in a reftest + graph (i.e. if it contains any <link rel=[mis]match>""" + return bool(self.references) + + def manifest_items(self): + """List of manifest items corresponding to the file. There is typically one + per test, but in the case of reftests a node may have corresponding manifest + items without being a test itself.""" + + if self.name_is_non_test: + rv = [] + + elif self.name_is_stub: + rv = [Stub(self, self.url)] + + elif self.name_is_manual: + rv = [ManualTest(self, self.url)] + + elif self.name_is_multi_global: + rv = [ + TestharnessTest(self, replace_end(self.url, ".any.js", ".any.html")), + TestharnessTest(self, replace_end(self.url, ".any.js", ".any.worker")), + ] + + elif self.name_is_worker: + rv = [TestharnessTest(self, replace_end(self.url, ".worker.js", ".worker"), + timeout=self.timeout)] + + elif self.name_is_webdriver: + rv = [WebdriverSpecTest(self, self.url)] + + elif self.content_is_testharness: + rv = [] + for variant in self.test_variants: + url = self.url + variant + rv.append(TestharnessTest(self, url, timeout=self.timeout)) + + elif self.content_is_ref_node: + rv = [RefTest(self, self.url, self.references, timeout=self.timeout, + viewport_size=self.viewport_size, dpi=self.dpi)] + + else: + # If nothing else it's a helper file, which we don't have a specific type for + rv = [] + + return rv diff --git a/testing/web-platform/tests/tools/manifest/tests/__init__.py b/testing/web-platform/tests/tools/manifest/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/tests/__init__.py diff --git a/testing/web-platform/tests/tools/manifest/tests/test_manifest.py b/testing/web-platform/tests/tools/manifest/tests/test_manifest.py new file mode 100644 index 000000000..11ac6d331 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/tests/test_manifest.py @@ -0,0 +1,80 @@ +from .. import manifest, item as manifestitem, sourcefile + + +def test_local_reftest_add(): + m = manifest.Manifest() + s = sourcefile.SourceFile("/", "test", "/") + test = manifestitem.RefTest(s, "/test", [("/ref", "==")]) + m.local_changes.add(test) + m.update_reftests() + assert list(m) == [(test.path, {test})] + + +def test_local_reftest_delete_path(): + m = manifest.Manifest() + s = sourcefile.SourceFile("/", "test", "/") + test = manifestitem.RefTest(s, "/test", [("/ref", "==")]) + m.add(test) + m.local_changes.add_deleted(test.path) + m.update_reftests() + assert list(m) == [] + + +def test_local_reftest_adjusted(): + m = manifest.Manifest() + s = sourcefile.SourceFile("/", "test", "/") + test = manifestitem.RefTest(s, "/test", [("/ref", "==")]) + m.add(test) + m.update_reftests() + + assert m.compute_reftests({test.path: {test}}) == {test} + + assert list(m) == [(test.path, {test})] + + s_1 = sourcefile.SourceFile("/", "test-1", "/") + test_1 = manifestitem.RefTest(s_1, "/test-1", [("/test", "==")]) + m.local_changes.add(test_1) + m.update_reftests() + + assert m.compute_reftests({test.path: {test}, test_1.path: {test_1}}) == {test_1} + + assert list(m) == [(test_1.path, {test_1})] + + +def test_manifest_to_json(): + m = manifest.Manifest() + s = sourcefile.SourceFile("/", "test", "/") + test = manifestitem.RefTest(s, "/test", [("/ref", "==")]) + m.add(test) + s_1 = sourcefile.SourceFile("/", "test-1", "/") + test_1 = manifestitem.RefTest(s_1, "/test-1", [("/test", "==")]) + m.local_changes.add(test_1) + m.local_changes.add_deleted(test.path) + m.update_reftests() + + json_str = m.to_json() + loaded = manifest.Manifest.from_json("/", json_str) + + assert list(loaded) == list(m) + + assert loaded.to_json() == json_str + + +def test_reftest_computation_chain(): + m = manifest.Manifest() + + s1 = sourcefile.SourceFile("/", "test1", "/") + s2 = sourcefile.SourceFile("/", "test2", "/") + + test1 = manifestitem.RefTest(s1, "/test1", [("/test3", "==")]) + test2 = manifestitem.RefTest(s2, "/test2", [("/test1", "==")]) + m.add(test1) + m.add(test2) + + m.update_reftests() + + assert m.reftest_nodes == {'test1': {test1}, + 'test2': {test2}} + + assert list(m) == [("test2", {test2})] + assert list(m.local_changes.itertypes()) == [] diff --git a/testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py b/testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py new file mode 100644 index 000000000..da51406c7 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py @@ -0,0 +1,251 @@ +from ..sourcefile import SourceFile + +def create(filename, contents=b""): + assert isinstance(contents, bytes) + return SourceFile("/", filename, "/", contents=contents) + + +def items(s): + return [ + (item.item_type, item.url) + for item in s.manifest_items() + ] + + +def test_name_is_non_test(): + non_tests = [ + ".gitignore", + ".travis.yml", + "MANIFEST.json", + "tools/test.html", + "resources/test.html", + "common/test.html", + "conformance-checkers/test.html", + ] + + for rel_path in non_tests: + s = create(rel_path) + assert s.name_is_non_test + + assert not s.content_is_testharness + + assert items(s) == [] + + +def test_name_is_manual(): + manual_tests = [ + "html/test-manual.html", + "html/test-manual.xhtml", + "html/test-manual.https.html", + "html/test-manual.https.xhtml" + ] + + for rel_path in manual_tests: + s = create(rel_path) + assert not s.name_is_non_test + assert s.name_is_manual + + assert not s.content_is_testharness + + assert items(s) == [("manual", "/" + rel_path)] + + +def test_worker(): + s = create("html/test.worker.js") + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert s.name_is_worker + assert not s.name_is_reference + + assert not s.content_is_testharness + + assert items(s) == [("testharness", "/html/test.worker")] + +def test_worker_long_timeout(): + s = create("html/test.worker.js", + contents="""// <meta> timeout=long +importScripts('/resources/testharnes.js') +test()""") + + manifest_items = s.manifest_items() + assert len(manifest_items) == 1 + assert manifest_items[0].timeout == "long" + + +def test_multi_global(): + s = create("html/test.any.js") + assert not s.name_is_non_test + assert not s.name_is_manual + assert s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert not s.content_is_testharness + + assert items(s) == [ + ("testharness", "/html/test.any.html"), + ("testharness", "/html/test.any.worker"), + ] + + +def test_testharness(): + content = b"<script src=/resources/testharness.js></script>" + + for ext in ["htm", "html"]: + filename = "html/test." + ext + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert s.content_is_testharness + + assert items(s) == [("testharness", "/" + filename)] + + +def test_relative_testharness(): + content = b"<script src=../resources/testharness.js></script>" + + for ext in ["htm", "html"]: + filename = "html/test." + ext + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert not s.content_is_testharness + + assert items(s) == [] + + +def test_testharness_xhtml(): + content = b""" +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +</head> +<body/> +</html> +""" + + for ext in ["xhtml", "xht", "xml"]: + filename = "html/test." + ext + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert s.content_is_testharness + + assert items(s) == [("testharness", "/" + filename)] + + +def test_relative_testharness_xhtml(): + content = b""" +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> +<script src="../resources/testharness.js"></script> +<script src="../resources/testharnessreport.js"></script> +</head> +<body/> +</html> +""" + + for ext in ["xhtml", "xht", "xml"]: + filename = "html/test." + ext + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert not s.content_is_testharness + + assert items(s) == [] + + +def test_testharness_svg(): + content = b"""\ +<?xml version="1.0" encoding="UTF-8"?> +<svg xmlns="http://www.w3.org/2000/svg" + xmlns:h="http://www.w3.org/1999/xhtml" + version="1.1" + width="100%" height="100%" viewBox="0 0 400 400"> +<title>Null test</title> +<h:script src="/resources/testharness.js"/> +<h:script src="/resources/testharnessreport.js"/> +</svg> +""" + + filename = "html/test.svg" + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert s.root + assert s.content_is_testharness + + assert items(s) == [("testharness", "/" + filename)] + + +def test_relative_testharness_svg(): + content = b"""\ +<?xml version="1.0" encoding="UTF-8"?> +<svg xmlns="http://www.w3.org/2000/svg" + xmlns:h="http://www.w3.org/1999/xhtml" + version="1.1" + width="100%" height="100%" viewBox="0 0 400 400"> +<title>Null test</title> +<h:script src="../resources/testharness.js"/> +<h:script src="../resources/testharnessreport.js"/> +</svg> +""" + + filename = "html/test.svg" + s = create(filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert s.root + assert not s.content_is_testharness + + assert items(s) == [] + + +def test_testharness_ext(): + content = b"<script src=/resources/testharness.js></script>" + + for filename in ["test", "test.test"]: + s = create("html/" + filename, content) + + assert not s.name_is_non_test + assert not s.name_is_manual + assert not s.name_is_multi_global + assert not s.name_is_worker + assert not s.name_is_reference + + assert not s.root + assert not s.content_is_testharness + + assert items(s) == [] diff --git a/testing/web-platform/tests/tools/manifest/tests/test_utils.py b/testing/web-platform/tests/tools/manifest/tests/test_utils.py new file mode 100644 index 000000000..a95bd47e9 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/tests/test_utils.py @@ -0,0 +1,28 @@ +import pytest + +from ..utils import is_blacklisted + + +@pytest.mark.parametrize("url", [ + "/foo", + "/tools/foo", + "/common/foo", + "/conformance-checkers/foo", + "/_certs/foo", + "/resources/foo", + "/support/foo", + "/foo/resources/bar", + "/foo/support/bar" +]) +def test_is_blacklisted(url): + assert is_blacklisted(url) is True + + +@pytest.mark.parametrize("url", [ + "/foo/tools/bar", + "/foo/common/bar", + "/foo/conformance-checkers/bar", + "/foo/_certs/bar" +]) +def test_not_is_blacklisted(url): + assert is_blacklisted(url) is False diff --git a/testing/web-platform/tests/tools/manifest/tree.py b/testing/web-platform/tests/tools/manifest/tree.py new file mode 100644 index 000000000..25a5f212f --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/tree.py @@ -0,0 +1,168 @@ +import os +from six.moves import cStringIO as StringIO +from fnmatch import fnmatch + +from . import vcs +from .log import get_logger +from .utils import is_blacklisted, rel_path_to_url + +def chunks(data, n): + for i in range(0, len(data) - 1, n): + yield data[i:i+n] + +class TestTree(object): + def __init__(self, tests_root, url_base): + self.tests_root = tests_root + self.url_base = url_base + self.logger = get_logger() + + def current_rev(self): + pass + + def local_changes(self): + pass + + def committed_changes(self, base_rev=None): + pass + + +class GitTree(TestTree): + def __init__(self, tests_root, url_base): + TestTree.__init__(self, tests_root, url_base) + self.git = self.setup_git() + + def setup_git(self): + assert vcs.is_git_repo(self.tests_root) + return vcs.get_git_func(self.tests_root) + + def current_rev(self): + return self.git("rev-parse", "HEAD").strip() + + def local_changes(self, path=None): + # -z is stable like --porcelain; see the git status documentation for details + cmd = ["status", "-z", "--ignore-submodules=all"] + if path is not None: + cmd.extend(["--", path]) + + rv = {} + + data = self.git(*cmd) + if data == "": + return rv + + assert data[-1] == "\0" + f = StringIO(data) + + while f.tell() < len(data): + # First two bytes are the status in the stage (index) and working tree, respectively + staged = f.read(1) + worktree = f.read(1) + assert f.read(1) == " " + + if staged == "R": + # When a file is renamed, there are two files, the source and the destination + files = 2 + else: + files = 1 + + filenames = [] + + for i in range(files): + filenames.append("") + char = f.read(1) + while char != "\0": + filenames[-1] += char + char = f.read(1) + + if not is_blacklisted(rel_path_to_url(filenames[0], self.url_base)): + rv.update(self.local_status(staged, worktree, filenames)) + + return rv + + def committed_changes(self, base_rev=None): + if base_rev is None: + self.logger.debug("Adding all changesets to the manifest") + return [(item, "modified") for item in self.paths()] + + self.logger.debug("Updating the manifest from %s to %s" % (base_rev, self.current_rev())) + rv = [] + data = self.git("diff", "-z", "--name-status", base_rev + "..HEAD") + items = data.split("\0") + for status, filename in chunks(items, 2): + if is_blacklisted(rel_path_to_url(filename, self.url_base)): + continue + if status == "D": + rv.append((filename, "deleted")) + else: + rv.append((filename, "modified")) + return rv + + def paths(self): + data = self.git("ls-tree", "--name-only", "--full-tree", "-r", "HEAD") + return [item for item in data.split("\n") if not item.endswith(os.path.sep)] + + def local_status(self, staged, worktree, filenames): + # Convert the complex range of statuses that git can have to two values + # we care about; "modified" and "deleted" and return a dictionary mapping + # filenames to statuses + + rv = {} + + if (staged, worktree) in [("D", "D"), ("A", "U"), ("U", "D"), ("U", "A"), + ("D", "U"), ("A", "A"), ("U", "U")]: + raise Exception("Can't operate on tree containing unmerged paths") + + if staged == "R": + assert len(filenames) == 2 + dest, src = filenames + rv[dest] = "modified" + rv[src] = "deleted" + else: + assert len(filenames) == 1 + + filename = filenames[0] + + if staged == "D" or worktree == "D": + # Actually if something is deleted in the index but present in the worktree + # it will get included by having a status of both "D " and "??". + # It isn't clear whether that's a bug + rv[filename] = "deleted" + elif staged == "?" and worktree == "?": + # A new file. If it's a directory, recurse into it + if os.path.isdir(os.path.join(self.tests_root, filename)): + if filename[-1] != '/': + filename += '/' + rv.update(self.local_changes(filename)) + else: + rv[filename] = "modified" + else: + rv[filename] = "modified" + + return rv + +class NoVCSTree(TestTree): + """Subclass that doesn't depend on git""" + + ignore = ["*.py[c|0]", "*~", "#*"] + + def current_rev(self): + return None + + def local_changes(self): + # Put all files into local_changes and rely on Manifest.update to de-dupe + # changes that in fact committed at the base rev. + + rv = [] + for dir_path, dir_names, filenames in os.walk(self.tests_root): + for filename in filenames: + if any(fnmatch(filename, pattern) for pattern in self.ignore): + continue + rel_path = os.path.relpath(os.path.join(dir_path, filename), + self.tests_root) + if is_blacklisted(rel_path_to_url(rel_path, self.url_base)): + continue + rv.append((rel_path, "modified")) + return dict(rv) + + def committed_changes(self, base_rev=None): + return None diff --git a/testing/web-platform/tests/tools/manifest/update.py b/testing/web-platform/tests/tools/manifest/update.py new file mode 100644 index 000000000..8460af257 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/update.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +import argparse +import imp +import os +import sys + +import manifest +from . import vcs +from .log import get_logger +from .tree import GitTree, NoVCSTree + +here = os.path.dirname(__file__) + +def update(tests_root, url_base, manifest, ignore_local=False): + if vcs.is_git_repo(tests_root): + tests_tree = GitTree(tests_root, url_base) + remove_missing_local = False + else: + tests_tree = NoVCSTree(tests_root, url_base) + remove_missing_local = not ignore_local + + if not ignore_local: + local_changes = tests_tree.local_changes() + else: + local_changes = None + + manifest.update(tests_root, + url_base, + tests_tree.current_rev(), + tests_tree.committed_changes(manifest.rev), + local_changes, + remove_missing_local=remove_missing_local) + + +def update_from_cli(**kwargs): + tests_root = kwargs["tests_root"] + path = kwargs["path"] + assert tests_root is not None + + m = None + logger = get_logger() + + if not kwargs.get("rebuild", False): + try: + m = manifest.load(tests_root, path) + except manifest.ManifestVersionMismatch: + logger.info("Manifest version changed, rebuilding") + m = None + else: + logger.info("Updating manifest") + + if m is None: + m = manifest.Manifest(None) + + + update(tests_root, + kwargs["url_base"], + m, + ignore_local=kwargs.get("ignore_local", False)) + manifest.write(m, path) + + +def abs_path(path): + return os.path.abspath(os.path.expanduser(path)) + + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-p", "--path", type=abs_path, help="Path to manifest file.") + parser.add_argument( + "--tests-root", type=abs_path, help="Path to root of tests.") + parser.add_argument( + "-r", "--rebuild", action="store_true", default=False, + help="Force a full rebuild of the manifest.") + parser.add_argument( + "--ignore-local", action="store_true", default=False, + help="Don't include uncommitted local changes in the manifest.") + parser.add_argument( + "--url-base", action="store", default="/", + help="Base url to use as the mount point for tests in this manifest.") + return parser + + +def find_top_repo(): + path = here + rv = None + while path != "/": + if vcs.is_git_repo(path): + rv = path + path = os.path.abspath(os.path.join(path, os.pardir)) + + return rv + +def main(default_tests_root=None): + opts = create_parser().parse_args() + + if opts.tests_root is None: + tests_root = None + if default_tests_root is not None: + tests_root = default_tests_root + else: + tests_root = find_top_repo() + + if tests_root is None: + print >> sys.stderr, """No git repo found; could not determine test root. +Run again with --test-root""" + sys.exit(1) + + opts.tests_root = tests_root + + if opts.path is None: + opts.path = os.path.join(opts.tests_root, "MANIFEST.json") + + update_from_cli(**vars(opts)) + + +if __name__ == "__main__": + main() diff --git a/testing/web-platform/tests/tools/manifest/utils.py b/testing/web-platform/tests/tools/manifest/utils.py new file mode 100644 index 000000000..c6b27229c --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/utils.py @@ -0,0 +1,52 @@ +import os +from six import BytesIO + +blacklist = ["/tools/", "/resources/", "/common/", "/conformance-checkers/", "/_certs/"] +blacklist_in = ["/resources/", "/support/"] + +def rel_path_to_url(rel_path, url_base="/"): + assert not os.path.isabs(rel_path) + if url_base[0] != "/": + url_base = "/" + url_base + if url_base[-1] != "/": + url_base += "/" + return url_base + rel_path.replace(os.sep, "/") + +def is_blacklisted(url): + if "/" not in url[1:]: + return True + for item in blacklist: + if url.startswith(item): + return True + for item in blacklist_in: + if item in url: + return True + return False + +def from_os_path(path): + return path.replace(os.path.sep, "/") + +def to_os_path(path): + return path.replace("/", os.path.sep) + +class ContextManagerBytesIO(BytesIO): + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + +class cached_property(object): + def __init__(self, func): + self.func = func + self.__doc__ = getattr(func, "__doc__") + self.name = func.__name__ + + def __get__(self, obj, cls=None): + if obj is None: + return self + + if self.name not in obj.__dict__: + obj.__dict__[self.name] = self.func(obj) + obj.__dict__.setdefault("__cached_properties__", set()).add(self.name) + return obj.__dict__[self.name] diff --git a/testing/web-platform/tests/tools/manifest/vcs.py b/testing/web-platform/tests/tools/manifest/vcs.py new file mode 100644 index 000000000..93bd445e1 --- /dev/null +++ b/testing/web-platform/tests/tools/manifest/vcs.py @@ -0,0 +1,25 @@ +import os +import subprocess + +def get_git_func(repo_path): + def git(cmd, *args): + full_cmd = ["git", cmd] + list(args) + return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT) + return git + + +def is_git_repo(tests_root): + return os.path.exists(os.path.join(tests_root, ".git")) + + +_repo_root = None +def get_repo_root(initial_dir=None): + global _repo_root + + if initial_dir is None: + initial_dir = os.path.dirname(__file__) + + if _repo_root is None: + git = get_git_func(initial_dir) + _repo_root = git("rev-parse", "--show-toplevel").rstrip() + return _repo_root |