summaryrefslogtreecommitdiffstats
path: root/js/src/tests/lib/manifest.py
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/tests/lib/manifest.py')
-rw-r--r--js/src/tests/lib/manifest.py395
1 files changed, 395 insertions, 0 deletions
diff --git a/js/src/tests/lib/manifest.py b/js/src/tests/lib/manifest.py
new file mode 100644
index 000000000..7079a186f
--- /dev/null
+++ b/js/src/tests/lib/manifest.py
@@ -0,0 +1,395 @@
+# Library for JSTest manifests.
+#
+# This includes classes for representing and parsing JS manifests.
+
+from __future__ import print_function
+
+import os, re, sys
+from subprocess import Popen, PIPE
+
+from tests import RefTestCase
+
+
+def split_path_into_dirs(path):
+ dirs = [path]
+
+ while True:
+ path, tail = os.path.split(path)
+ if not tail:
+ break
+ dirs.append(path)
+ return dirs
+
+class XULInfo:
+ def __init__(self, abi, os, isdebug):
+ self.abi = abi
+ self.os = os
+ self.isdebug = isdebug
+ self.browserIsRemote = False
+
+ def as_js(self):
+ """Return JS that when executed sets up variables so that JS expression
+ predicates on XUL build info evaluate properly."""
+
+ return ('var xulRuntime = {{ OS: "{}", XPCOMABI: "{}", shell: true }};'
+ 'var isDebugBuild={}; var Android={}; '
+ 'var browserIsRemote={}'.format(
+ self.os,
+ self.abi,
+ str(self.isdebug).lower(),
+ str(self.os == "Android").lower(),
+ str(self.browserIsRemote).lower()))
+
+ @classmethod
+ def create(cls, jsdir):
+ """Create a XULInfo based on the current platform's characteristics."""
+
+ # Our strategy is to find the autoconf.mk generated for the build and
+ # read the values from there.
+
+ # Find config/autoconf.mk.
+ dirs = split_path_into_dirs(os.getcwd()) + split_path_into_dirs(jsdir)
+
+ path = None
+ for dir in dirs:
+ _path = os.path.join(dir, 'config/autoconf.mk')
+ if os.path.isfile(_path):
+ path = _path
+ break
+
+ if path == None:
+ print("Can't find config/autoconf.mk on a directory containing"
+ " the JS shell (searched from {})".format(jsdir))
+ sys.exit(1)
+
+ # Read the values.
+ val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)')
+ kw = {'isdebug': False}
+ for line in open(path):
+ m = val_re.match(line)
+ if m:
+ key, val = m.groups()
+ val = val.rstrip()
+ if key == 'TARGET_XPCOM_ABI':
+ kw['abi'] = val
+ if key == 'OS_TARGET':
+ kw['os'] = val
+ if key == 'MOZ_DEBUG':
+ kw['isdebug'] = (val == '1')
+ return cls(**kw)
+
+class XULInfoTester:
+ def __init__(self, xulinfo, js_bin):
+ self.js_prologue = xulinfo.as_js()
+ self.js_bin = js_bin
+ # Maps JS expr to evaluation result.
+ self.cache = {}
+
+ def test(self, cond):
+ """Test a XUL predicate condition against this local info."""
+ ans = self.cache.get(cond, None)
+ if ans is None:
+ cmd = [
+ self.js_bin,
+ # run in safe configuration, since it is hard to debug
+ # crashes when running code here. In particular, msan will
+ # error out if the jit is active.
+ '--no-baseline',
+ '-e', self.js_prologue,
+ '-e', 'print(!!({}))'.format(cond)
+ ]
+ p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ out, err = p.communicate()
+ if out in ('true\n', 'true\r\n'):
+ ans = True
+ elif out in ('false\n', 'false\r\n'):
+ ans = False
+ else:
+ raise Exception("Failed to test XUL condition {!r};"
+ " output was {!r}, stderr was {!r}".format(
+ cond, out, err))
+ self.cache[cond] = ans
+ return ans
+
+class NullXULInfoTester:
+ """Can be used to parse manifests without a JS shell."""
+ def test(self, cond):
+ return False
+
+def _parse_one(testcase, xul_tester):
+ pos = 0
+ parts = testcase.terms.split()
+ while pos < len(parts):
+ if parts[pos] == 'fails':
+ testcase.expect = False
+ pos += 1
+ elif parts[pos] == 'skip':
+ testcase.expect = testcase.enable = False
+ pos += 1
+ elif parts[pos] == 'random':
+ testcase.random = True
+ pos += 1
+ elif parts[pos].startswith('fails-if'):
+ cond = parts[pos][len('fails-if('):-1]
+ if xul_tester.test(cond):
+ testcase.expect = False
+ pos += 1
+ elif parts[pos].startswith('asserts-if'):
+ # This directive means we may flunk some number of
+ # NS_ASSERTIONs in the browser. For the shell, ignore it.
+ pos += 1
+ elif parts[pos].startswith('skip-if'):
+ cond = parts[pos][len('skip-if('):-1]
+ if xul_tester.test(cond):
+ testcase.expect = testcase.enable = False
+ pos += 1
+ elif parts[pos].startswith('random-if'):
+ cond = parts[pos][len('random-if('):-1]
+ if xul_tester.test(cond):
+ testcase.random = True
+ pos += 1
+ elif parts[pos] == 'slow':
+ testcase.slow = True
+ pos += 1
+ elif parts[pos] == 'silentfail':
+ # silentfails use tons of memory, and Darwin doesn't support ulimit.
+ if xul_tester.test("xulRuntime.OS == 'Darwin'"):
+ testcase.expect = testcase.enable = False
+ pos += 1
+ else:
+ print('warning: invalid manifest line element "{}"'.format(
+ parts[pos]))
+ pos += 1
+
+def _build_manifest_script_entry(script_name, test):
+ line = []
+ if test.terms:
+ line.append(test.terms)
+ line.append("script")
+ line.append(script_name)
+ if test.comment:
+ line.append("#")
+ line.append(test.comment)
+ return ' '.join(line)
+
+def _map_prefixes_left(test_gen):
+ """
+ Splits tests into a dictionary keyed on the first component of the test
+ path, aggregating tests with a common base path into a list.
+ """
+ byprefix = {}
+ for t in test_gen:
+ left, sep, remainder = t.path.partition(os.sep)
+ if left not in byprefix:
+ byprefix[left] = []
+ if remainder:
+ t.path = remainder
+ byprefix[left].append(t)
+ return byprefix
+
+def _emit_manifest_at(location, relative, test_gen, depth):
+ """
+ location - str: absolute path where we want to write the manifest
+ relative - str: relative path from topmost manifest directory to current
+ test_gen - (str): generator of all test paths and directorys
+ depth - int: number of dirs we are below the topmost manifest dir
+ """
+ manifests = _map_prefixes_left(test_gen)
+
+ filename = os.path.join(location, 'jstests.list')
+ manifest = []
+ numTestFiles = 0
+ for k, test_list in manifests.iteritems():
+ fullpath = os.path.join(location, k)
+ if os.path.isdir(fullpath):
+ manifest.append("include " + k + "/jstests.list")
+ relpath = os.path.join(relative, k)
+ _emit_manifest_at(fullpath, relpath, test_list, depth + 1)
+ else:
+ numTestFiles += 1
+ if len(test_list) != 1:
+ import pdb; pdb.set_trace()
+ assert len(test_list) == 1
+ line = _build_manifest_script_entry(k, test_list[0])
+ manifest.append(line)
+
+ # Always present our manifest in sorted order.
+ manifest.sort()
+
+ # If we have tests, we have to set the url-prefix so reftest can find them.
+ if numTestFiles > 0:
+ manifest = ["url-prefix {}jsreftest.html?test={}/".format(
+ '../' * depth, relative)] + manifest
+
+ fp = open(filename, 'w')
+ try:
+ fp.write('\n'.join(manifest) + '\n')
+ finally:
+ fp.close()
+
+def make_manifests(location, test_gen):
+ _emit_manifest_at(location, '', test_gen, 0)
+
+def _find_all_js_files(base, location):
+ for root, dirs, files in os.walk(location):
+ root = root[len(base) + 1:]
+ for fn in files:
+ if fn.endswith('.js'):
+ yield root, fn
+
+TEST_HEADER_PATTERN_INLINE = re.compile(r'//\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?$')
+TEST_HEADER_PATTERN_MULTI = re.compile(r'/\*\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?\*/')
+
+def _parse_test_header(fullpath, testcase, xul_tester):
+ """
+ This looks a bit weird. The reason is that it needs to be efficient, since
+ it has to be done on every test
+ """
+ fp = open(fullpath, 'r')
+ try:
+ buf = fp.read(512)
+ finally:
+ fp.close()
+
+ # Bail early if we do not start with a single comment.
+ if not buf.startswith("//"):
+ return
+
+ # Extract the token.
+ buf, _, _ = buf.partition('\n')
+ matches = TEST_HEADER_PATTERN_INLINE.match(buf)
+
+ if not matches:
+ matches = TEST_HEADER_PATTERN_MULTI.match(buf)
+ if not matches:
+ return
+
+ testcase.tag = matches.group(1)
+ testcase.terms = matches.group(2)
+ testcase.comment = matches.group(4)
+
+ _parse_one(testcase, xul_tester)
+
+def _parse_external_manifest(filename, relpath):
+ """
+ Reads an external manifest file for test suites whose individual test cases
+ can't be decorated with reftest comments.
+ filename - str: name of the manifest file
+ relpath - str: relative path of the directory containing the manifest
+ within the test suite
+ """
+ entries = []
+
+ with open(filename, 'r') as fp:
+ manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
+ for line in fp:
+ line, _, comment = line.partition('#')
+ line = line.strip()
+ if not line:
+ continue
+ matches = manifest_re.match(line)
+ if not matches:
+ print('warning: unrecognized line in jstests.list:'
+ ' {0}'.format(line))
+ continue
+
+ path = os.path.normpath(os.path.join(relpath, matches.group(3)))
+ if matches.group(2) == 'include':
+ # The manifest spec wants a reference to another manifest here,
+ # but we need just the directory. We do need the trailing
+ # separator so we don't accidentally match other paths of which
+ # this one is a prefix.
+ assert(path.endswith('jstests.list'))
+ path = path[:-len('jstests.list')]
+
+ entries.append({'path': path, 'terms': matches.group(1),
+ 'comment': comment.strip()})
+
+ # if one directory name is a prefix of another, we want the shorter one
+ # first
+ entries.sort(key=lambda x: x["path"])
+ return entries
+
+def _apply_external_manifests(filename, testcase, entries, xul_tester):
+ for entry in entries:
+ if filename.startswith(entry["path"]):
+ # The reftest spec would require combining the terms (failure types)
+ # that may already be defined in the test case with the terms
+ # specified in entry; for example, a skip overrides a random, which
+ # overrides a fails. Since we don't necessarily know yet in which
+ # environment the test cases will be run, we'd also have to
+ # consider skip-if, random-if, and fails-if with as-yet unresolved
+ # conditions.
+ # At this point, we use external manifests only for test cases
+ # that can't have their own failure type comments, so we simply
+ # use the terms for the most specific path.
+ testcase.terms = entry["terms"]
+ testcase.comment = entry["comment"]
+ _parse_one(testcase, xul_tester)
+
+def _is_test_file(path_from_root, basename, filename, requested_paths,
+ excluded_paths):
+ # Any file whose basename matches something in this set is ignored.
+ EXCLUDED = set(('browser.js', 'shell.js', 'template.js',
+ 'user.js', 'sta.js',
+ 'test262-browser.js', 'test262-shell.js',
+ 'test402-browser.js', 'test402-shell.js',
+ 'testBuiltInObject.js', 'testIntl.js',
+ 'js-test-driver-begin.js', 'js-test-driver-end.js'))
+
+ # Skip js files in the root test directory.
+ if not path_from_root:
+ return False
+
+ # Skip files that we know are not tests.
+ if basename in EXCLUDED:
+ return False
+
+ # If any tests are requested by name, skip tests that do not match.
+ if requested_paths \
+ and not any(req in filename for req in requested_paths):
+ return False
+
+ # Skip excluded tests.
+ if filename in excluded_paths:
+ return False
+
+ return True
+
+
+def count_tests(location, requested_paths, excluded_paths):
+ count = 0
+ for root, basename in _find_all_js_files(location, location):
+ filename = os.path.join(root, basename)
+ if _is_test_file(root, basename, filename, requested_paths, excluded_paths):
+ count += 1
+ return count
+
+
+def load_reftests(location, requested_paths, excluded_paths, xul_tester, reldir=''):
+ """
+ Locates all tests by walking the filesystem starting at |location|.
+ Uses xul_tester to evaluate any test conditions in the test header.
+ Failure type and comment for a test case can come from
+ - an external manifest entry for the test case,
+ - an external manifest entry for a containing directory,
+ - most commonly: the header of the test case itself.
+ """
+ manifestFile = os.path.join(location, 'jstests.list')
+ externalManifestEntries = _parse_external_manifest(manifestFile, '')
+
+ for root, basename in _find_all_js_files(location, location):
+ # Get the full path and relative location of the file.
+ filename = os.path.join(root, basename)
+ if not _is_test_file(root, basename, filename, requested_paths, excluded_paths):
+ continue
+
+ # Skip empty files.
+ fullpath = os.path.join(location, filename)
+ statbuf = os.stat(fullpath)
+
+ testcase = RefTestCase(os.path.join(reldir, filename))
+ _apply_external_manifests(filename, testcase, externalManifestEntries,
+ xul_tester)
+ _parse_test_header(fullpath, testcase, xul_tester)
+ yield testcase