diff options
Diffstat (limited to 'js/src/tests/lib/jittests.py')
| -rwxr-xr-x | js/src/tests/lib/jittests.py | 720 |
1 files changed, 720 insertions, 0 deletions
diff --git a/js/src/tests/lib/jittests.py b/js/src/tests/lib/jittests.py new file mode 100755 index 000000000..0a1d1537a --- /dev/null +++ b/js/src/tests/lib/jittests.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +# jit_test.py -- Python harness for JavaScript trace tests. + +from __future__ import print_function +import os, posixpath, sys, tempfile, traceback, time +import subprocess +from collections import namedtuple +import StringIO + +if sys.platform.startswith('linux') or sys.platform.startswith('darwin'): + from tasks_unix import run_all_tests +else: + from tasks_win import run_all_tests + +from progressbar import ProgressBar, NullProgressBar +from results import TestOutput + +TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__)) +JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR)) +TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR)) +TEST_DIR = os.path.join(JS_DIR, 'jit-test', 'tests') +LIB_DIR = os.path.join(JS_DIR, 'jit-test', 'lib') + os.path.sep +MODULE_DIR = os.path.join(JS_DIR, 'jit-test', 'modules') + os.path.sep +JS_CACHE_DIR = os.path.join(JS_DIR, 'jit-test', '.js-cache') +JS_TESTS_DIR = posixpath.join(JS_DIR, 'tests') + +# Backported from Python 3.1 posixpath.py +def _relpath(path, start=None): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + + if start is None: + start = os.curdir + + start_list = os.path.abspath(start).split(os.sep) + path_list = os.path.abspath(path).split(os.sep) + + # Work out how much of the filepath is shared by start and path. + i = len(os.path.commonprefix([start_list, path_list])) + + rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return os.curdir + return os.path.join(*rel_list) + +# Mapping of Python chars to their javascript string representation. +QUOTE_MAP = { + '\\': '\\\\', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', + '\v': '\\v' +} + +# Quote the string S, javascript style. +def js_quote(quote, s): + result = quote + for c in s: + if c == quote: + result += '\\' + quote + elif c in QUOTE_MAP: + result += QUOTE_MAP[c] + else: + result += c + result += quote + return result + +os.path.relpath = _relpath + +class JitTest: + + VALGRIND_CMD = [] + paths = (d for d in os.environ['PATH'].split(os.pathsep)) + valgrinds = (os.path.join(d, 'valgrind') for d in paths) + if any(os.path.exists(p) for p in valgrinds): + VALGRIND_CMD = [ + 'valgrind', '-q', '--smc-check=all-non-file', + '--error-exitcode=1', '--gen-suppressions=all', + '--show-possibly-lost=no', '--leak-check=full', + ] + if os.uname()[0] == 'Darwin': + VALGRIND_CMD.append('--dsymutil=yes') + + del paths + del valgrinds + + def __init__(self, path): + # Absolute path of the test file. + self.path = path + + # Path relative to the top mozilla/ directory. + self.relpath_top = os.path.relpath(path, TOP_SRC_DIR) + + # Path relative to mozilla/js/src/jit-test/tests/. + self.relpath_tests = os.path.relpath(path, TEST_DIR) + + self.jitflags = [] # jit flags to enable + self.slow = False # True means the test is slow-running + self.allow_oom = False # True means that OOM is not considered a failure + self.allow_unhandlable_oom = False # True means CrashAtUnhandlableOOM + # is not considered a failure + self.allow_overrecursed = False # True means that hitting recursion the + # limits is not considered a failure. + self.valgrind = False # True means run under valgrind + self.tz_pacific = False # True means force Pacific time for the test + self.test_also_noasmjs = False # True means run with and without asm.js + # enabled. + self.test_also_wasm_baseline = False # True means run with and and without + # wasm baseline compiler enabled. + self.test_also = [] # List of other configurations to test with. + self.test_join = [] # List of other configurations to test with all existing variants. + self.expect_error = '' # Errors to expect and consider passing + self.expect_status = 0 # Exit status to expect from shell + self.expect_crash = False # Exit status or error output. + self.is_module = False + self.test_reflect_stringify = None # Reflect.stringify implementation to test + + # Expected by the test runner. Always true for jit-tests. + self.enable = True + + def copy(self): + t = JitTest(self.path) + t.jitflags = self.jitflags[:] + t.slow = self.slow + t.allow_oom = self.allow_oom + t.allow_unhandlable_oom = self.allow_unhandlable_oom + t.allow_overrecursed = self.allow_overrecursed + t.valgrind = self.valgrind + t.tz_pacific = self.tz_pacific + t.test_also_noasmjs = self.test_also_noasmjs + t.test_also_wasm_baseline = self.test_also_noasmjs + t.test_also = self.test_also + t.test_join = self.test_join + t.expect_error = self.expect_error + t.expect_status = self.expect_status + t.expect_crash = self.expect_crash + t.test_reflect_stringify = self.test_reflect_stringify + t.enable = True + t.is_module = self.is_module + return t + + def copy_and_extend_jitflags(self, variant): + t = self.copy() + t.jitflags.extend(variant) + return t + + def copy_variants(self, variants): + # Append variants to be tested in addition to the current set of tests. + variants = variants + self.test_also + + # For each existing variant, duplicates it for each list of options in + # test_join. This will multiply the number of variants by 2 for set of + # options. + for join_opts in self.test_join: + variants = variants + [ opts + join_opts for opts in variants ]; + + # For each list of jit flags, make a copy of the test. + return [self.copy_and_extend_jitflags(v) for v in variants] + + + COOKIE = '|jit-test|' + CacheDir = JS_CACHE_DIR + Directives = {} + + @classmethod + def find_directives(cls, file_name): + meta = '' + line = open(file_name).readline() + i = line.find(cls.COOKIE) + if i != -1: + meta = ';' + line[i + len(cls.COOKIE):].strip('\n') + return meta + + @classmethod + def from_file(cls, path, options): + test = cls(path) + + # If directives.txt exists in the test's directory then it may + # contain metainformation that will be catenated with + # whatever's in the test file. The form of the directive in + # the directive file is the same as in the test file. Only + # the first line is considered, just as for the test file. + + dir_meta = '' + dir_name = os.path.dirname(path) + if dir_name in cls.Directives: + dir_meta = cls.Directives[dir_name] + else: + meta_file_name = os.path.join(dir_name, "directives.txt") + if os.path.exists(meta_file_name): + dir_meta = cls.find_directives(meta_file_name) + cls.Directives[dir_name] = dir_meta + + meta = cls.find_directives(path) + if meta != '' or dir_meta != '': + meta = meta + dir_meta + parts = meta.split(';') + for part in parts: + part = part.strip() + if not part: + continue + name, _, value = part.partition(':') + if value: + value = value.strip() + if name == 'error': + test.expect_error = value + elif name == 'exitstatus': + try: + test.expect_status = int(value, 0) + except ValueError: + print("warning: couldn't parse exit status" + " {}".format(value)) + elif name == 'thread-count': + try: + test.jitflags.append('--thread-count={}'.format( + int(value, 0))) + except ValueError: + print("warning: couldn't parse thread-count" + " {}".format(value)) + else: + print('{}: warning: unrecognized |jit-test| attribute' + ' {}'.format(path, part)) + else: + if name == 'slow': + test.slow = True + elif name == 'allow-oom': + test.allow_oom = True + elif name == 'allow-unhandlable-oom': + test.allow_unhandlable_oom = True + elif name == 'allow-overrecursed': + test.allow_overrecursed = True + elif name == 'valgrind': + test.valgrind = options.valgrind + elif name == 'tz-pacific': + test.tz_pacific = True + elif name == 'test-also-noasmjs': + if options.asmjs_enabled: + test.test_also.append(['--no-asmjs']) + elif name == 'test-also-wasm-baseline': + if options.wasm_enabled: + test.test_also.append(['--wasm-always-baseline']) + elif name == 'test-also-wasm-check-bce': + if options.wasm_enabled: + test.test_also.append(['--wasm-check-bce']) + elif name.startswith('test-also='): + test.test_also.append([name[len('test-also='):]]) + elif name.startswith('test-join='): + test.test_join.append([name[len('test-join='):]]) + elif name == 'module': + test.is_module = True + elif name == 'crash': + test.expect_crash = True + elif name.startswith('--'): + # // |jit-test| --ion-gvn=off; --no-sse4 + test.jitflags.append(name) + else: + print('{}: warning: unrecognized |jit-test| attribute' + ' {}'.format(path, part)) + + if options.valgrind_all: + test.valgrind = True + + if options.test_reflect_stringify is not None: + test.expect_error = '' + test.expect_status = 0 + + return test + + def command(self, prefix, libdir, moduledir, remote_prefix=None): + path = self.path + if remote_prefix: + path = self.path.replace(TEST_DIR, remote_prefix) + + scriptdir_var = os.path.dirname(path) + if not scriptdir_var.endswith('/'): + scriptdir_var += '/' + + # Platforms where subprocess immediately invokes exec do not care + # whether we use double or single quotes. On windows and when using + # a remote device, however, we have to be careful to use the quote + # style that is the opposite of what the exec wrapper uses. + if remote_prefix: + quotechar = '"' + else: + quotechar = "'" + expr = "const platform={}; const libdir={}; const scriptdir={}".format( + js_quote(quotechar, sys.platform), + js_quote(quotechar, libdir), + js_quote(quotechar, scriptdir_var)) + + # We may have specified '-a' or '-d' twice: once via --jitflags, once + # via the "|jit-test|" line. Remove dups because they are toggles. + cmd = prefix + ['--js-cache', JitTest.CacheDir] + cmd += list(set(self.jitflags)) + ['-e', expr] + if self.is_module: + cmd += ['--module-load-path', moduledir] + cmd += ['--module', path] + elif self.test_reflect_stringify is None: + cmd += ['-f', path] + else: + cmd += ['--', self.test_reflect_stringify, "--check", path] + if self.valgrind: + cmd = self.VALGRIND_CMD + cmd + return cmd + + # The test runner expects this to be set to give to get_command. + js_cmd_prefix = None + def get_command(self, prefix): + """Shim for the test runner.""" + return self.command(prefix, LIB_DIR, MODULE_DIR) + + +def find_tests(substring=None): + ans = [] + for dirpath, dirnames, filenames in os.walk(TEST_DIR): + dirnames.sort() + filenames.sort() + if dirpath == '.': + continue + for filename in filenames: + if not filename.endswith('.js'): + continue + if filename in ('shell.js', 'browser.js'): + continue + test = os.path.join(dirpath, filename) + if substring is None \ + or substring in os.path.relpath(test, TEST_DIR): + ans.append(test) + return ans + +def run_test_remote(test, device, prefix, options): + if options.test_reflect_stringify: + raise ValueError("can't run Reflect.stringify tests remotely") + cmd = test.command(prefix, + posixpath.join(options.remote_test_root, 'lib/'), + posixpath.join(options.remote_test_root, 'modules/'), + posixpath.join(options.remote_test_root, 'tests')) + if options.show_cmd: + print(subprocess.list2cmdline(cmd)) + + env = {} + if test.tz_pacific: + env['TZ'] = 'PST8PDT' + + env['LD_LIBRARY_PATH'] = options.remote_test_root + + buf = StringIO.StringIO() + returncode = device.shell(cmd, buf, env=env, cwd=options.remote_test_root, + timeout=int(options.timeout)) + + out = buf.getvalue() + # We can't distinguish between stdout and stderr so we pass + # the same buffer to both. + return TestOutput(test, cmd, out, out, returncode, None, False) + +def check_output(out, err, rc, timed_out, test, options): + if timed_out: + if test.relpath_tests in options.ignore_timeouts: + return True + + # The shell sometimes hangs on shutdown on Windows 7 and Windows + # Server 2008. See bug 970063 comment 7 for a description of the + # problem. Until bug 956899 is fixed, ignore timeouts on these + # platforms (versions 6.0 and 6.1). + if sys.platform == 'win32': + ver = sys.getwindowsversion() + if ver.major == 6 and ver.minor <= 1: + return True + return False + + if test.expect_error: + # The shell exits with code 3 on uncaught exceptions. + # Sometimes 0 is returned on Windows for unknown reasons. + # See bug 899697. + if sys.platform in ['win32', 'cygwin']: + if rc != 3 and rc != 0: + return False + else: + if rc != 3: + return False + + return test.expect_error in err + + for line in out.split('\n'): + if line.startswith('Trace stats check failed'): + return False + + for line in err.split('\n'): + if 'Assertion failed:' in line: + return False + + if test.expect_crash: + if sys.platform == 'win32' and rc == 3 - 2 ** 31: + return True + + if sys.platform != 'win32' and rc == -11: + return True + + # When building with ASan enabled, ASan will convert the -11 returned + # value to 1. As a work-around we look for the error output which + # includes the crash reason. + if rc == 1 and ("Hit MOZ_CRASH" in err or "Assertion failure:" in err): + return True + + if rc != test.expect_status: + # Tests which expect a timeout check for exit code 6. + # Sometimes 0 is returned on Windows for unknown reasons. + # See bug 899697. + if sys.platform in ['win32', 'cygwin'] and rc == 0: + return True + + # Allow a non-zero exit code if we want to allow OOM, but only if we + # actually got OOM. + if test.allow_oom and 'out of memory' in err \ + and 'Assertion failure' not in err and 'MOZ_CRASH' not in err: + return True + + # Allow a non-zero exit code if we want to allow unhandlable OOM, but + # only if we actually got unhandlable OOM. + if test.allow_unhandlable_oom \ + and 'Assertion failure: [unhandlable oom]' in err: + return True + + # Allow a non-zero exit code if we want to all too-much-recursion and + # the test actually over-recursed. + if test.allow_overrecursed and 'too much recursion' in err \ + and 'Assertion failure' not in err: + return True + + return False + + return True + +def print_automation_format(ok, res): + # Output test failures in a parsable format suitable for automation, eg: + # TEST-RESULT | filename.js | Failure description (code N, args "--foobar") + # + # Example: + # TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager") + # TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion") + # INFO exit-status : 3 + # INFO timed-out : False + # INFO stdout > foo + # INFO stdout > bar + # INFO stdout > baz + # INFO stderr 2> TypeError: or something + # TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user + result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL" + message = "Success" if ok else res.describe_failure() + jitflags = " ".join(res.test.jitflags) + print("{} | {} | {} (code {}, args \"{}\")".format( + result, res.test.relpath_top, message, res.rc, jitflags)) + + # For failed tests, print as much information as we have, to aid debugging. + if ok: + return + print("INFO exit-status : {}".format(res.rc)) + print("INFO timed-out : {}".format(res.timed_out)) + for line in res.out.splitlines(): + print("INFO stdout > " + line.strip()) + for line in res.err.splitlines(): + print("INFO stderr 2> " + line.strip()) + +def print_test_summary(num_tests, failures, complete, doing, options): + if failures: + if options.write_failures: + try: + out = open(options.write_failures, 'w') + # Don't write duplicate entries when we are doing multiple + # failures per job. + written = set() + for res in failures: + if res.test.path not in written: + out.write(os.path.relpath(res.test.path, TEST_DIR) + + '\n') + if options.write_failure_output: + out.write(res.out) + out.write(res.err) + out.write('Exit code: ' + str(res.rc) + "\n") + written.add(res.test.path) + out.close() + except IOError: + sys.stderr.write("Exception thrown trying to write failure" + " file '{}'\n".format(options.write_failures)) + traceback.print_exc() + sys.stderr.write('---\n') + + def show_test(res): + if options.show_failed: + print(' ' + subprocess.list2cmdline(res.cmd)) + else: + print(' ' + ' '.join(res.test.jitflags + [res.test.path])) + + print('FAILURES:') + for res in failures: + if not res.timed_out: + show_test(res) + + print('TIMEOUTS:') + for res in failures: + if res.timed_out: + show_test(res) + else: + print('PASSED ALL' + + ('' if complete + else ' (partial run -- interrupted by user {})'.format(doing))) + + if options.format == 'automation': + num_failures = len(failures) if failures else 0 + print('Result summary:') + print('Passed: {:d}'.format(num_tests - num_failures)) + print('Failed: {:d}'.format(num_failures)) + + return not failures + +def create_progressbar(num_tests, options): + if not options.hide_progress and not options.show_cmd \ + and ProgressBar.conservative_isatty(): + fmt = [ + {'value': 'PASS', 'color': 'green'}, + {'value': 'FAIL', 'color': 'red'}, + {'value': 'TIMEOUT', 'color': 'blue'}, + {'value': 'SKIP', 'color': 'brightgray'}, + ] + return ProgressBar(num_tests, fmt) + return NullProgressBar() + +def process_test_results(results, num_tests, pb, options): + failures = [] + timeouts = 0 + complete = False + output_dict = {} + doing = 'before starting' + + if num_tests == 0: + pb.finish(True) + complete = True + return print_test_summary(num_tests, failures, complete, doing, options) + + try: + for i, res in enumerate(results): + ok = check_output(res.out, res.err, res.rc, res.timed_out, + res.test, options) + + if ok: + show_output = options.show_output and not options.failed_only + else: + show_output = options.show_output or not options.no_show_failed + + if show_output: + pb.beginline() + sys.stdout.write(res.out) + sys.stdout.write(res.err) + sys.stdout.write('Exit code: {}\n'.format(res.rc)) + + if res.test.valgrind and not show_output: + pb.beginline() + sys.stdout.write(res.err) + + if options.check_output: + if res.test.path in output_dict.keys(): + if output_dict[res.test.path] != res.out: + pb.message("FAIL - OUTPUT DIFFERS {}".format(res.test.relpath_tests)) + else: + output_dict[res.test.path] = res.out + + doing = 'after {}'.format(res.test.relpath_tests) + if not ok: + failures.append(res) + if res.timed_out: + pb.message("TIMEOUT - {}".format(res.test.relpath_tests)) + timeouts += 1 + else: + pb.message("FAIL - {}".format(res.test.relpath_tests)) + + if options.format == 'automation': + print_automation_format(ok, res) + + n = i + 1 + pb.update(n, { + 'PASS': n - len(failures), + 'FAIL': len(failures), + 'TIMEOUT': timeouts, + 'SKIP': 0 + }) + complete = True + except KeyboardInterrupt: + print("TEST-UNEXPECTED-FAIL | jit_test.py" + + " : Test execution interrupted by user") + + pb.finish(True) + return print_test_summary(num_tests, failures, complete, doing, options) + +def run_tests(tests, num_tests, prefix, options): + # The jstests tasks runner requires the following options. The names are + # taken from the jstests options processing code, which are frequently + # subtly different from the options jit-tests expects. As such, we wrap + # them here, as needed. + AdaptorOptions = namedtuple("AdaptorOptions", [ + "worker_count", "passthrough", "timeout", "output_fp", + "hide_progress", "run_skipped", "show_cmd"]) + shim_options = AdaptorOptions(options.max_jobs, False, options.timeout, + sys.stdout, False, True, options.show_cmd) + + # The test runner wants the prefix as a static on the Test class. + JitTest.js_cmd_prefix = prefix + + pb = create_progressbar(num_tests, options) + gen = run_all_tests(tests, prefix, pb, shim_options) + ok = process_test_results(gen, num_tests, pb, options) + return ok + +def get_remote_results(tests, device, prefix, options): + from mozdevice import devicemanager + + try: + for i in xrange(0, options.repeat): + for test in tests: + yield run_test_remote(test, device, prefix, options) + except devicemanager.DMError as e: + # After a devicemanager error, the device is typically in a + # state where all further tests will fail so there is no point in + # continuing here. + sys.stderr.write("Error running remote tests: {}".format(e.message)) + +def push_libs(options, device): + # This saves considerable time in pushing unnecessary libraries + # to the device but needs to be updated if the dependencies change. + required_libs = ['libnss3.so', 'libmozglue.so', 'libnspr4.so', + 'libplc4.so', 'libplds4.so'] + + for file in os.listdir(options.local_lib): + if file in required_libs: + remote_file = posixpath.join(options.remote_test_root, file) + device.pushFile(os.path.join(options.local_lib, file), remote_file) + +def push_progs(options, device, progs): + for local_file in progs: + remote_file = posixpath.join(options.remote_test_root, + os.path.basename(local_file)) + device.pushFile(local_file, remote_file) + +def run_tests_remote(tests, num_tests, prefix, options): + # Setup device with everything needed to run our tests. + from mozdevice import devicemanagerADB, devicemanagerSUT + + if options.device_transport == 'adb': + if options.device_ip: + dm = devicemanagerADB.DeviceManagerADB( + options.device_ip, options.device_port, + deviceSerial=options.device_serial, + packageName=None, + deviceRoot=options.remote_test_root) + else: + dm = devicemanagerADB.DeviceManagerADB( + deviceSerial=options.device_serial, + packageName=None, + deviceRoot=options.remote_test_root) + else: + dm = devicemanagerSUT.DeviceManagerSUT( + options.device_ip, options.device_port, + deviceRoot=options.remote_test_root) + if options.device_ip == None: + print('Error: you must provide a device IP to connect to via the' + ' --device option') + sys.exit(1) + + # Update the test root to point to our test directory. + jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests') + options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests') + + # Push js shell and libraries. + if dm.dirExists(jit_tests_dir): + dm.removeDir(jit_tests_dir) + dm.mkDirs(options.remote_test_root) + push_libs(options, dm) + push_progs(options, dm, [prefix[0]]) + dm.chmodDir(options.remote_test_root) + + JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache') + dm.mkDir(JitTest.CacheDir) + + dm.pushDir(JS_TESTS_DIR, posixpath.join(jit_tests_dir, 'tests'), + timeout=600) + + dm.pushDir(os.path.dirname(TEST_DIR), options.remote_test_root, + timeout=600) + prefix[0] = os.path.join(options.remote_test_root, 'js') + + # Run all tests. + pb = create_progressbar(num_tests, options) + gen = get_remote_results(tests, dm, prefix, options) + ok = process_test_results(gen, num_tests, pb, options) + return ok + +def platform_might_be_android(): + try: + # The python package for SL4A provides an |android| module. + # If that module is present, we're likely in SL4A-python on + # device. False positives and negatives are possible, + # however. + import android + return True + except ImportError: + return False + +def stdio_might_be_broken(): + return platform_might_be_android() + +if __name__ == '__main__': + print('Use ../jit-test/jit_test.py to run these tests.') |