#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# jit_test.py -- Python harness for JavaScript trace tests.
from __future__ import print_function
import os, posixpath, sys, tempfile, traceback, time
import subprocess
from collections import namedtuple
import StringIO
if sys.platform.startswith('linux') or sys.platform.startswith('darwin'):
from tasks_unix import run_all_tests
else:
from tasks_win import run_all_tests
from progressbar import ProgressBar, NullProgressBar
from results import TestOutput
TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__))
JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR))
TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR))
TEST_DIR = os.path.join(JS_DIR, 'jit-test', 'tests')
LIB_DIR = os.path.join(JS_DIR, 'jit-test', 'lib') + os.path.sep
MODULE_DIR = os.path.join(JS_DIR, 'jit-test', 'modules') + os.path.sep
JS_CACHE_DIR = os.path.join(JS_DIR, 'jit-test', '.js-cache')
JS_TESTS_DIR = posixpath.join(JS_DIR, 'tests')
# Backported from Python 3.1 posixpath.py
def _relpath(path, start=None):
"""Return a relative version of a path"""
if not path:
raise ValueError("no path specified")
if start is None:
start = os.curdir
start_list = os.path.abspath(start).split(os.sep)
path_list = os.path.abspath(path).split(os.sep)
# Work out how much of the filepath is shared by start and path.
i = len(os.path.commonprefix([start_list, path_list]))
rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
if not rel_list:
return os.curdir
return os.path.join(*rel_list)
# Mapping of Python chars to their javascript string representation.
QUOTE_MAP = {
'\\': '\\\\',
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t',
'\v': '\\v'
}
# Quote the string S, javascript style.
def js_quote(quote, s):
result = quote
for c in s:
if c == quote:
result += '\\' + quote
elif c in QUOTE_MAP:
result += QUOTE_MAP[c]
else:
result += c
result += quote
return result
os.path.relpath = _relpath
class JitTest:
VALGRIND_CMD = []
paths = (d for d in os.environ['PATH'].split(os.pathsep))
valgrinds = (os.path.join(d, 'valgrind') for d in paths)
if any(os.path.exists(p) for p in valgrinds):
VALGRIND_CMD = [
'valgrind', '-q', '--smc-check=all-non-file',
'--error-exitcode=1', '--gen-suppressions=all',
'--show-possibly-lost=no', '--leak-check=full',
]
if os.uname()[0] == 'Darwin':
VALGRIND_CMD.append('--dsymutil=yes')
del paths
del valgrinds
def __init__(self, path):
# Absolute path of the test file.
self.path = path
# Path relative to the top mozilla/ directory.
self.relpath_top = os.path.relpath(path, TOP_SRC_DIR)
# Path relative to mozilla/js/src/jit-test/tests/.
self.relpath_tests = os.path.relpath(path, TEST_DIR)
self.jitflags = [] # jit flags to enable
self.slow = False # True means the test is slow-running
self.allow_oom = False # True means that OOM is not considered a failure
self.allow_unhandlable_oom = False # True means CrashAtUnhandlableOOM
# is not considered a failure
self.allow_overrecursed = False # True means that hitting recursion the
# limits is not considered a failure.
self.valgrind = False # True means run under valgrind
self.tz_pacific = False # True means force Pacific time for the test
self.test_also_noasmjs = False # True means run with and without asm.js
# enabled.
self.test_also_wasm_baseline = False # True means run with and and without
# wasm baseline compiler enabled.
self.test_also = [] # List of other configurations to test with.
self.test_join = [] # List of other configurations to test with all existing variants.
self.expect_error = '' # Errors to expect and consider passing
self.expect_status = 0 # Exit status to expect from shell
self.expect_crash = False # Exit status or error output.
self.is_module = False
self.test_reflect_stringify = None # Reflect.stringify implementation to test
# Expected by the test runner. Always true for jit-tests.
self.enable = True
def copy(self):
t = JitTest(self.path)
t.jitflags = self.jitflags[:]
t.slow = self.slow
t.allow_oom = self.allow_oom
t.allow_unhandlable_oom = self.allow_unhandlable_oom
t.allow_overrecursed = self.allow_overrecursed
t.valgrind = self.valgrind
t.tz_pacific = self.tz_pacific
t.test_also_noasmjs = self.test_also_noasmjs
t.test_also_wasm_baseline = self.test_also_noasmjs
t.test_also = self.test_also
t.test_join = self.test_join
t.expect_error = self.expect_error
t.expect_status = self.expect_status
t.expect_crash = self.expect_crash
t.test_reflect_stringify = self.test_reflect_stringify
t.enable = True
t.is_module = self.is_module
return t
def copy_and_extend_jitflags(self, variant):
t = self.copy()
t.jitflags.extend(variant)
return t
def copy_variants(self, variants):
# Append variants to be tested in addition to the current set of tests.
variants = variants + self.test_also
# For each existing variant, duplicates it for each list of options in
# test_join. This will multiply the number of variants by 2 for set of
# options.
for join_opts in self.test_join:
variants = variants + [ opts + join_opts for opts in variants ];
# For each list of jit flags, make a copy of the test.
return [self.copy_and_extend_jitflags(v) for v in variants]
COOKIE = '|jit-test|'
CacheDir = JS_CACHE_DIR
Directives = {}
@classmethod
def find_directives(cls, file_name):
meta = ''
line = open(file_name).readline()
i = line.find(cls.COOKIE)
if i != -1:
meta = ';' + line[i + len(cls.COOKIE):].strip('\n')
return meta
@classmethod
def from_file(cls, path, options):
test = cls(path)
# If directives.txt exists in the test's directory then it may
# contain metainformation that will be catenated with
# whatever's in the test file. The form of the directive in
# the directive file is the same as in the test file. Only
# the first line is considered, just as for the test file.
dir_meta = ''
dir_name = os.path.dirname(path)
if dir_name in cls.Directives:
dir_meta = cls.Directives[dir_name]
else:
meta_file_name = os.path.join(dir_name, "directives.txt")
if os.path.exists(meta_file_name):
dir_meta = cls.find_directives(meta_file_name)
cls.Directives[dir_name] = dir_meta
meta = cls.find_directives(path)
if meta != '' or dir_meta != '':
meta = meta + dir_meta
parts = meta.split(';')
for part in parts:
part = part.strip()
if not part:
continue
name, _, value = part.partition(':')
if value:
value = value.strip()
if name == 'error':
test.expect_error = value
elif name == 'exitstatus':
try:
test.expect_status = int(value, 0)
except ValueError:
print("warning: couldn't parse exit status"
" {}".format(value))
elif name == 'thread-count':
try:
test.jitflags.append('--thread-count={}'.format(
int(value, 0)))
except ValueError:
print("warning: couldn't parse thread-count"
" {}".format(value))
else:
print('{}: warning: unrecognized |jit-test| attribute'
' {}'.format(path, part))
else:
if name == 'slow':
test.slow = True
elif name == 'allow-oom':
test.allow_oom = True
elif name == 'allow-unhandlable-oom':
test.allow_unhandlable_oom = True
elif name == 'allow-overrecursed':
test.allow_overrecursed = True
elif name == 'valgrind':
test.valgrind = options.valgrind
elif name == 'tz-pacific':
test.tz_pacific = True
elif name == 'test-also-noasmjs':
if options.asmjs_enabled:
test.test_also.append(['--no-asmjs'])
elif name == 'test-also-wasm-baseline':
if options.wasm_enabled:
test.test_also.append(['--wasm-always-baseline'])
elif name == 'test-also-wasm-check-bce':
if options.wasm_enabled:
test.test_also.append(['--wasm-check-bce'])
elif name.startswith('test-also='):
test.test_also.append([name[len('test-also='):]])
elif name.startswith('test-join='):
test.test_join.append([name[len('test-join='):]])
elif name == 'module':
test.is_module = True
elif name == 'crash':
test.expect_crash = True
elif name.startswith('--'):
# // |jit-test| --ion-gvn=off; --no-sse4
test.jitflags.append(name)
else:
print('{}: warning: unrecognized |jit-test| attribute'
' {}'.format(path, part))
if options.valgrind_all:
test.valgrind = True
if options.test_reflect_stringify is not None:
test.expect_error = ''
test.expect_status = 0
return test
def command(self, prefix, libdir, moduledir, remote_prefix=None):
path = self.path
if remote_prefix:
path = self.path.replace(TEST_DIR, remote_prefix)
scriptdir_var = os.path.dirname(path)
if not scriptdir_var.endswith('/'):
scriptdir_var += '/'
# Platforms where subprocess immediately invokes exec do not care
# whether we use double or single quotes. On windows and when using
# a remote device, however, we have to be careful to use the quote
# style that is the opposite of what the exec wrapper uses.
if remote_prefix:
quotechar = '"'
else:
quotechar = "'"
expr = "const platform={}; const libdir={}; const scriptdir={}".format(
js_quote(quotechar, sys.platform),
js_quote(quotechar, libdir),
js_quote(quotechar, scriptdir_var))
# We may have specified '-a' or '-d' twice: once via --jitflags, once
# via the "|jit-test|" line. Remove dups because they are toggles.
cmd = prefix + ['--js-cache', JitTest.CacheDir]
cmd += list(set(self.jitflags)) + ['-e', expr]
if self.is_module:
cmd += ['--module-load-path', moduledir]
cmd += ['--module', path]
elif self.test_reflect_stringify is None:
cmd += ['-f', path]
else:
cmd += ['--', self.test_reflect_stringify, "--check", path]
if self.valgrind:
cmd = self.VALGRIND_CMD + cmd
return cmd
# The test runner expects this to be set to give to get_command.
js_cmd_prefix = None
def get_command(self, prefix):
"""Shim for the test runner."""
return self.command(prefix, LIB_DIR, MODULE_DIR)
def find_tests(substring=None):
ans = []
for dirpath, dirnames, filenames in os.walk(TEST_DIR):
dirnames.sort()
filenames.sort()
if dirpath == '.':
continue
for filename in filenames:
if not filename.endswith('.js'):
continue
if filename in ('shell.js', 'browser.js'):
continue
test = os.path.join(dirpath, filename)
if substring is None \
or substring in os.path.relpath(test, TEST_DIR):
ans.append(test)
return ans
def run_test_remote(test, device, prefix, options):
if options.test_reflect_stringify:
raise ValueError("can't run Reflect.stringify tests remotely")
cmd = test.command(prefix,
posixpath.join(options.remote_test_root, 'lib/'),
posixpath.join(options.remote_test_root, 'modules/'),
posixpath.join(options.remote_test_root, 'tests'))
if options.show_cmd:
print(subprocess.list2cmdline(cmd))
env = {}
if test.tz_pacific:
env['TZ'] = 'PST8PDT'
env['LD_LIBRARY_PATH'] = options.remote_test_root
buf = StringIO.StringIO()
returncode = device.shell(cmd, buf, env=env, cwd=options.remote_test_root,
timeout=int(options.timeout))
out = buf.getvalue()
# We can't distinguish between stdout and stderr so we pass
# the same buffer to both.
return TestOutput(test, cmd, out, out, returncode, None, False)
def check_output(out, err, rc, timed_out, test, options):
if timed_out:
if test.relpath_tests in options.ignore_timeouts:
return True
# The shell sometimes hangs on shutdown on Windows 7 and Windows
# Server 2008. See bug 970063 comment 7 for a description of the
# problem. Until bug 956899 is fixed, ignore timeouts on these
# platforms (versions 6.0 and 6.1).
if sys.platform == 'win32':
ver = sys.getwindowsversion()
if ver.major == 6 and ver.minor <= 1:
return True
return False
if test.expect_error:
# The shell exits with code 3 on uncaught exceptions.
# Sometimes 0 is returned on Windows for unknown reasons.
# See bug 899697.
if sys.platform in ['win32', 'cygwin']:
if rc != 3 and rc != 0:
return False
else:
if rc != 3:
return False
return test.expect_error in err
for line in out.split('\n'):
if line.startswith('Trace stats check failed'):
return False
for line in err.split('\n'):
if 'Assertion failed:' in line:
return False
if test.expect_crash:
if sys.platform == 'win32' and rc == 3 - 2 ** 31:
return True
if sys.platform != 'win32' and rc == -11:
return True
# When building with ASan enabled, ASan will convert the -11 returned
# value to 1. As a work-around we look for the error output which
# includes the crash reason.
if rc == 1 and ("Hit MOZ_CRASH" in err or "Assertion failure:" in err):
return True
if rc != test.expect_status:
# Tests which expect a timeout check for exit code 6.
# Sometimes 0 is returned on Windows for unknown reasons.
# See bug 899697.
if sys.platform in ['win32', 'cygwin'] and rc == 0:
return True
# Allow a non-zero exit code if we want to allow OOM, but only if we
# actually got OOM.
if test.allow_oom and 'out of memory' in err \
and 'Assertion failure' not in err and 'MOZ_CRASH' not in err:
return True
# Allow a non-zero exit code if we want to allow unhandlable OOM, but
# only if we actually got unhandlable OOM.
if test.allow_unhandlable_oom \
and 'Assertion failure: [unhandlable oom]' in err:
return True
# Allow a non-zero exit code if we want to all too-much-recursion and
# the test actually over-recursed.
if test.allow_overrecursed and 'too much recursion' in err \
and 'Assertion failure' not in err:
return True
return False
return True
def print_automation_format(ok, res):
# Output test failures in a parsable format suitable for automation, eg:
# TEST-RESULT | filename.js | Failure description (code N, args "--foobar")
#
# Example:
# TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager")
# TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion")
# INFO exit-status : 3
# INFO timed-out : False
# INFO stdout > foo
# INFO stdout > bar
# INFO stdout > baz
# INFO stderr 2> TypeError: or something
# TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user
result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL"
message = "Success" if ok else res.describe_failure()
jitflags = " ".join(res.test.jitflags)
print("{} | {} | {} (code {}, args \"{}\")".format(
result, res.test.relpath_top, message, res.rc, jitflags))
# For failed tests, print as much information as we have, to aid debugging.
if ok:
return
print("INFO exit-status : {}".format(res.rc))
print("INFO timed-out : {}".format(res.timed_out))
for line in res.out.splitlines():
print("INFO stdout > " + line.strip())
for line in res.err.splitlines():
print("INFO stderr 2> " + line.strip())
def print_test_summary(num_tests, failures, complete, doing, options):
if failures:
if options.write_failures:
try:
out = open(options.write_failures, 'w')
# Don't write duplicate entries when we are doing multiple
# failures per job.
written = set()
for res in failures:
if res.test.path not in written:
out.write(os.path.relpath(res.test.path, TEST_DIR)
+ '\n')
if options.write_failure_output:
out.write(res.out)
out.write(res.err)
out.write('Exit code: ' + str(res.rc) + "\n")
written.add(res.test.path)
out.close()
except IOError:
sys.stderr.write("Exception thrown trying to write failure"
" file '{}'\n".format(options.write_failures))
traceback.print_exc()
sys.stderr.write('---\n')
def show_test(res):
if options.show_failed:
print(' ' + subprocess.list2cmdline(res.cmd))
else:
print(' ' + ' '.join(res.test.jitflags + [res.test.path]))
print('FAILURES:')
for res in failures:
if not res.timed_out:
show_test(res)
print('TIMEOUTS:')
for res in failures:
if res.timed_out:
show_test(res)
else:
print('PASSED ALL'
+ ('' if complete
else ' (partial run -- interrupted by user {})'.format(doing)))
if options.format == 'automation':
num_failures = len(failures) if failures else 0
print('Result summary:')
print('Passed: {:d}'.format(num_tests - num_failures))
print('Failed: {:d}'.format(num_failures))
return not failures
def create_progressbar(num_tests, options):
if not options.hide_progress and not options.show_cmd \
and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
return ProgressBar(num_tests, fmt)
return NullProgressBar()
def process_test_results(results, num_tests, pb, options):
failures = []
timeouts = 0
complete = False
output_dict = {}
doing = 'before starting'
if num_tests == 0:
pb.finish(True)
complete = True
return print_test_summary(num_tests, failures, complete, doing, options)
try:
for i, res in enumerate(results):
ok = check_output(res.out, res.err, res.rc, res.timed_out,
res.test, options)
if ok:
show_output = options.show_output and not options.failed_only
else:
show_output = options.show_output or not options.no_show_failed
if show_output:
pb.beginline()
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write('Exit code: {}\n'.format(res.rc))
if res.test.valgrind and not show_output:
pb.beginline()
sys.stdout.write(res.err)
if options.check_output:
if res.test.path in output_dict.keys():
if output_dict[res.test.path] != res.out:
pb.message("FAIL - OUTPUT DIFFERS {}".format(res.test.relpath_tests))
else:
output_dict[res.test.path] = res.out
doing = 'after {}'.format(res.test.relpath_tests)
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - {}".format(res.test.relpath_tests))
timeouts += 1
else:
pb.message("FAIL - {}".format(res.test.relpath_tests))
if options.format == 'automation':
print_automation_format(ok, res)
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0
})
complete = True
except KeyboardInterrupt:
print("TEST-UNEXPECTED-FAIL | jit_test.py" +
" : Test execution interrupted by user")
pb.finish(True)
return print_test_summary(num_tests, failures, complete, doing, options)
def run_tests(tests, num_tests, prefix, options):
# The jstests tasks runner requires the following options. The names are
# taken from the jstests options processing code, which are frequently
# subtly different from the options jit-tests expects. As such, we wrap
# them here, as needed.
AdaptorOptions = namedtuple("AdaptorOptions", [
"worker_count", "passthrough", "timeout", "output_fp",
"hide_progress", "run_skipped", "show_cmd"])
shim_options = AdaptorOptions(options.max_jobs, False, options.timeout,
sys.stdout, False, True, options.show_cmd)
# The test runner wants the prefix as a static on the Test class.
JitTest.js_cmd_prefix = prefix
pb = create_progressbar(num_tests, options)
gen = run_all_tests(tests, prefix, pb, shim_options)
ok = process_test_results(gen, num_tests, pb, options)
return ok
def get_remote_results(tests, device, prefix, options):
from mozdevice import devicemanager
try:
for i in xrange(0, options.repeat):
for test in tests:
yield run_test_remote(test, device, prefix, options)
except devicemanager.DMError as e:
# After a devicemanager error, the device is typically in a
# state where all further tests will fail so there is no point in
# continuing here.
sys.stderr.write("Error running remote tests: {}".format(e.message))
def push_libs(options, device):
# This saves considerable time in pushing unnecessary libraries
# to the device but needs to be updated if the dependencies change.
required_libs = ['libnss3.so', 'libmozglue.so', 'libnspr4.so',
'libplc4.so', 'libplds4.so']
for file in os.listdir(options.local_lib):
if file in required_libs:
remote_file = posixpath.join(options.remote_test_root, file)
device.pushFile(os.path.join(options.local_lib, file), remote_file)
def push_progs(options, device, progs):
for local_file in progs:
remote_file = posixpath.join(options.remote_test_root,
os.path.basename(local_file))
device.pushFile(local_file, remote_file)
def run_tests_remote(tests, num_tests, prefix, options):
# Setup device with everything needed to run our tests.
from mozdevice import devicemanagerADB, devicemanagerSUT
if options.device_transport == 'adb':
if options.device_ip:
dm = devicemanagerADB.DeviceManagerADB(
options.device_ip, options.device_port,
deviceSerial=options.device_serial,
packageName=None,
deviceRoot=options.remote_test_root)
else:
dm = devicemanagerADB.DeviceManagerADB(
deviceSerial=options.device_serial,
packageName=None,
deviceRoot=options.remote_test_root)
else:
dm = devicemanagerSUT.DeviceManagerSUT(
options.device_ip, options.device_port,
deviceRoot=options.remote_test_root)
if options.device_ip == None:
print('Error: you must provide a device IP to connect to via the'
' --device option')
sys.exit(1)
# Update the test root to point to our test directory.
jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
# Push js shell and libraries.
if dm.dirExists(jit_tests_dir):
dm.removeDir(jit_tests_dir)
dm.mkDirs(options.remote_test_root)
push_libs(options, dm)
push_progs(options, dm, [prefix[0]])
dm.chmodDir(options.remote_test_root)
JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache')
dm.mkDir(JitTest.CacheDir)
dm.pushDir(JS_TESTS_DIR, posixpath.join(jit_tests_dir, 'tests'),
timeout=600)
dm.pushDir(os.path.dirname(TEST_DIR), options.remote_test_root,
timeout=600)
prefix[0] = os.path.join(options.remote_test_root, 'js')
# Run all tests.
pb = create_progressbar(num_tests, options)
gen = get_remote_results(tests, dm, prefix, options)
ok = process_test_results(gen, num_tests, pb, options)
return ok
def platform_might_be_android():
try:
# The python package for SL4A provides an |android| module.
# If that module is present, we're likely in SL4A-python on
# device. False positives and negatives are possible,
# however.
import android
return True
except ImportError:
return False
def stdio_might_be_broken():
return platform_might_be_android()
if __name__ == '__main__':
print('Use ../jit-test/jit_test.py to run these tests.')