summaryrefslogtreecommitdiffstats
path: root/js/src/tests/jstests.py
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/tests/jstests.py')
-rwxr-xr-xjs/src/tests/jstests.py382
1 files changed, 382 insertions, 0 deletions
diff --git a/js/src/tests/jstests.py b/js/src/tests/jstests.py
new file mode 100755
index 000000000..6cd92a5e0
--- /dev/null
+++ b/js/src/tests/jstests.py
@@ -0,0 +1,382 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+The JS Shell Test Harness.
+
+See the adjacent README.txt for more details.
+"""
+
+from __future__ import print_function
+
+import os, sys, textwrap, platform
+from os.path import abspath, dirname, isfile, realpath
+from contextlib import contextmanager
+from copy import copy
+from subprocess import list2cmdline, call
+
+from lib.tests import RefTestCase, get_jitflags, get_cpu_count, \
+ get_environment_overlay, change_env
+from lib.results import ResultsSink
+from lib.progressbar import ProgressBar
+
+if sys.platform.startswith('linux') or sys.platform.startswith('darwin'):
+ from lib.tasks_unix import run_all_tests
+else:
+ from lib.tasks_win import run_all_tests
+
+
+@contextmanager
+def changedir(dirname):
+ pwd = os.getcwd()
+ os.chdir(dirname)
+ try:
+ yield
+ finally:
+ os.chdir(pwd)
+
+
+def parse_args():
+ """
+ Parse command line arguments.
+ Returns a tuple of: (options, js_shell, requested_paths, excluded_paths)
+ options :object: The raw OptionParser output.
+ js_shell :str: The absolute location of the shell to test with.
+ requested_paths :set<str>: Test paths specially requested on the CLI.
+ excluded_paths :set<str>: Test paths specifically excluded by the CLI.
+ """
+ from optparse import OptionParser, OptionGroup
+ op = OptionParser(usage=textwrap.dedent("""
+ %prog [OPTIONS] JS_SHELL [TESTS]
+
+ Shell output format: [ pass | fail | timeout | skip ] progress | time
+ """).strip())
+ op.add_option('--xul-info', dest='xul_info_src',
+ help='config data for xulRuntime'
+ ' (avoids search for config/autoconf.mk)')
+
+ harness_og = OptionGroup(op, "Harness Controls",
+ "Control how tests are run.")
+ harness_og.add_option('-j', '--worker-count', type=int,
+ default=max(1, get_cpu_count()),
+ help='Number of tests to run in parallel'
+ ' (default %default)')
+ harness_og.add_option('-t', '--timeout', type=float, default=150.0,
+ help='Set maximum time a test is allows to run'
+ ' (in seconds).')
+ harness_og.add_option('-a', '--args', dest='shell_args', default='',
+ help='Extra args to pass to the JS shell.')
+ harness_og.add_option('--jitflags', dest='jitflags', default='none',
+ type='string',
+ help='IonMonkey option combinations. One of all,'
+ ' debug, ion, and none (default %default).')
+ harness_og.add_option('--tbpl', action='store_true',
+ help='Runs each test in all configurations tbpl'
+ ' tests.')
+ harness_og.add_option('--tbpl-debug', action='store_true',
+ help='Runs each test in some faster configurations'
+ ' tbpl tests.')
+ harness_og.add_option('-g', '--debug', action='store_true',
+ help='Run a test in debugger.')
+ harness_og.add_option('--debugger', default='gdb -q --args',
+ help='Debugger command.')
+ harness_og.add_option('-J', '--jorendb', action='store_true',
+ help='Run under JS debugger.')
+ harness_og.add_option('--passthrough', action='store_true',
+ help='Run tests with stdin/stdout attached to'
+ ' caller.')
+ harness_og.add_option('--test-reflect-stringify', dest="test_reflect_stringify",
+ help="instead of running tests, use them to test the "
+ "Reflect.stringify code in specified file")
+ harness_og.add_option('--valgrind', action='store_true',
+ help='Run tests in valgrind.')
+ harness_og.add_option('--valgrind-args', default='',
+ help='Extra args to pass to valgrind.')
+ harness_og.add_option('--rr', action='store_true',
+ help='Run tests under RR record-and-replay debugger.')
+ harness_og.add_option('-C', '--check-output', action='store_true',
+ help='Run tests to check output for different jit-flags')
+ op.add_option_group(harness_og)
+
+ input_og = OptionGroup(op, "Inputs", "Change what tests are run.")
+ input_og.add_option('-f', '--file', dest='test_file', action='append',
+ help='Get tests from the given file.')
+ input_og.add_option('-x', '--exclude-file', action='append',
+ help='Exclude tests from the given file.')
+ input_og.add_option('-d', '--exclude-random', dest='random',
+ action='store_false',
+ help='Exclude tests marked as "random."')
+ input_og.add_option('--run-skipped', action='store_true',
+ help='Run tests marked as "skip."')
+ input_og.add_option('--run-only-skipped', action='store_true',
+ help='Run only tests marked as "skip."')
+ input_og.add_option('--run-slow-tests', action='store_true',
+ help='Do not skip tests marked as "slow."')
+ input_og.add_option('--no-extensions', action='store_true',
+ help='Run only tests conforming to the ECMAScript 5'
+ ' standard.')
+ input_og.add_option('--repeat', type=int, default=1,
+ help='Repeat tests the given number of times.')
+ op.add_option_group(input_og)
+
+ output_og = OptionGroup(op, "Output",
+ "Modify the harness and tests output.")
+ output_og.add_option('-s', '--show-cmd', action='store_true',
+ help='Show exact commandline used to run each test.')
+ output_og.add_option('-o', '--show-output', action='store_true',
+ help="Print each test's output to the file given by"
+ " --output-file.")
+ output_og.add_option('-F', '--failed-only', action='store_true',
+ help="If a --show-* option is given, only print"
+ " output for failed tests.")
+ output_og.add_option('--no-show-failed', action='store_true',
+ help="Don't print output for failed tests"
+ " (no-op with --show-output).")
+ output_og.add_option('-O', '--output-file',
+ help='Write all output to the given file'
+ ' (default: stdout).')
+ output_og.add_option('--failure-file',
+ help='Write all not-passed tests to the given file.')
+ output_og.add_option('--no-progress', dest='hide_progress',
+ action='store_true',
+ help='Do not show the progress bar.')
+ output_og.add_option('--tinderbox', dest='format', action='store_const',
+ const='automation',
+ help='Use automation-parseable output format.')
+ output_og.add_option('--format', dest='format', default='none',
+ type='choice', choices=['automation', 'none'],
+ help='Output format. Either automation or none'
+ ' (default %default).')
+ op.add_option_group(output_og)
+
+ special_og = OptionGroup(op, "Special",
+ "Special modes that do not run tests.")
+ special_og.add_option('--make-manifests', metavar='BASE_TEST_PATH',
+ help='Generate reftest manifest files.')
+ op.add_option_group(special_og)
+ options, args = op.parse_args()
+
+ # Acquire the JS shell given on the command line.
+ options.js_shell = None
+ requested_paths = set()
+ if len(args) > 0:
+ options.js_shell = abspath(args[0])
+ requested_paths |= set(args[1:])
+
+ # If we do not have a shell, we must be in a special mode.
+ if options.js_shell is None and not options.make_manifests:
+ op.error('missing JS_SHELL argument')
+
+ # Valgrind, gdb, and rr are mutually exclusive.
+ if sum(map(lambda e: 1 if e else 0, [options.valgrind, options.debug, options.rr])) > 1:
+ op.error("--valgrind, --debug, and --rr are mutually exclusive.")
+
+ # Fill the debugger field, as needed.
+ if options.debug:
+ if options.debugger == 'lldb':
+ debugger_prefix = ['lldb', '--']
+ else:
+ debugger_prefix = options.debugger.split()
+ else:
+ debugger_prefix = []
+
+ if options.valgrind:
+ debugger_prefix = ['valgrind'] + options.valgrind_args.split()
+ if os.uname()[0] == 'Darwin':
+ debugger_prefix.append('--dsymutil=yes')
+ options.show_output = True
+ if options.rr:
+ debugger_prefix = ['rr', 'record']
+
+ js_cmd_args = options.shell_args.split()
+ if options.jorendb:
+ options.passthrough = True
+ options.hide_progress = True
+ options.worker_count = 1
+ debugger_path = realpath(os.path.join(
+ abspath(dirname(abspath(__file__))),
+ '..', '..', 'examples', 'jorendb.js'))
+ js_cmd_args.extend(['-d', '-f', debugger_path, '--'])
+ prefix = RefTestCase.build_js_cmd_prefix(options.js_shell, js_cmd_args,
+ debugger_prefix)
+
+ # If files with lists of tests to run were specified, add them to the
+ # requested tests set.
+ if options.test_file:
+ for test_file in options.test_file:
+ requested_paths |= set(
+ [line.strip() for line in open(test_file).readlines()])
+
+ # If files with lists of tests to exclude were specified, add them to the
+ # excluded tests set.
+ excluded_paths = set()
+ if options.exclude_file:
+ for filename in options.exclude_file:
+ try:
+ fp = open(filename, 'r')
+ for line in fp:
+ if line.startswith('#'): continue
+ line = line.strip()
+ if not line: continue
+ excluded_paths |= set((line,))
+ finally:
+ fp.close()
+
+ # Handle output redirection, if requested and relevant.
+ options.output_fp = sys.stdout
+ if options.output_file:
+ if not options.show_cmd:
+ options.show_output = True
+ try:
+ options.output_fp = open(options.output_file, 'w')
+ except IOError as ex:
+ raise SystemExit("Failed to open output file: " + str(ex))
+
+ # Hide the progress bar if it will get in the way of other output.
+ options.hide_progress = (options.format == 'automation' or
+ not ProgressBar.conservative_isatty() or
+ options.hide_progress)
+
+ return (options, prefix, requested_paths, excluded_paths)
+
+
+def load_tests(options, requested_paths, excluded_paths):
+ """
+ Returns a tuple: (skipped_tests, test_list)
+ test_count: [int] Number of tests that will be in test_gen
+ test_gen: [iterable<Test>] Tests found that should be run.
+ """
+ import lib.manifest as manifest
+
+ if options.js_shell is None:
+ xul_tester = manifest.NullXULInfoTester()
+ else:
+ if options.xul_info_src is None:
+ xul_info = manifest.XULInfo.create(options.js_shell)
+ else:
+ xul_abi, xul_os, xul_debug = options.xul_info_src.split(r':')
+ xul_debug = xul_debug.lower() is 'true'
+ xul_info = manifest.XULInfo(xul_abi, xul_os, xul_debug)
+ xul_tester = manifest.XULInfoTester(xul_info, options.js_shell)
+
+ test_dir = dirname(abspath(__file__))
+ test_count = manifest.count_tests(test_dir, requested_paths, excluded_paths)
+ test_gen = manifest.load_reftests(test_dir, requested_paths, excluded_paths,
+ xul_tester)
+
+ if options.test_reflect_stringify is not None:
+ def trs_gen(tests):
+ for test in tests:
+ test.test_reflect_stringify = options.test_reflect_stringify
+ # Even if the test is not normally expected to pass, we still
+ # expect reflect-stringify to be able to handle it.
+ test.expect = True
+ test.random = False
+ test.slow = False
+ yield test
+ test_gen = trs_gen(test_gen)
+
+ if options.make_manifests:
+ manifest.make_manifests(options.make_manifests, test_gen)
+ sys.exit()
+
+ # Create a new test list. Apply each TBPL configuration to every test.
+ flags_list = None
+ if options.tbpl:
+ flags_list = get_jitflags('all')
+ elif options.tbpl_debug:
+ flags_list = get_jitflags('debug')
+ else:
+ flags_list = get_jitflags(options.jitflags, none=None)
+
+ if flags_list:
+ def flag_gen(tests):
+ for test in tests:
+ for jitflags in flags_list:
+ tmp_test = copy(test)
+ tmp_test.jitflags = copy(test.jitflags)
+ tmp_test.jitflags.extend(jitflags)
+ yield tmp_test
+ test_count = test_count * len(flags_list)
+ test_gen = flag_gen(test_gen)
+
+ if options.test_file:
+ paths = set()
+ for test_file in options.test_file:
+ paths |= set(
+ [line.strip() for line in open(test_file).readlines()])
+ test_gen = (_ for _ in test_gen if _.path in paths)
+
+ if options.no_extensions:
+ pattern = os.sep + 'extensions' + os.sep
+ test_gen = (_ for _ in test_gen if pattern not in _.path)
+
+ if not options.random:
+ test_gen = (_ for _ in test_gen if not _.random)
+
+ if options.run_only_skipped:
+ options.run_skipped = True
+ test_gen = (_ for _ in test_gen if not _.enable)
+
+ if not options.run_slow_tests:
+ test_gen = (_ for _ in test_gen if not _.slow)
+
+ if options.repeat:
+ test_gen = (test for test in test_gen for i in range(options.repeat))
+ test_count *= options.repeat
+
+ return test_count, test_gen
+
+
+def main():
+ options, prefix, requested_paths, excluded_paths = parse_args()
+ if options.js_shell is not None and not (isfile(options.js_shell) and
+ os.access(options.js_shell, os.X_OK)):
+ if (platform.system() != 'Windows' or
+ isfile(options.js_shell) or not
+ isfile(options.js_shell + ".exe") or not
+ os.access(options.js_shell + ".exe", os.X_OK)):
+ print('Could not find executable shell: ' + options.js_shell)
+ return 1
+
+ test_count, test_gen = load_tests(options, requested_paths, excluded_paths)
+ test_environment = get_environment_overlay(options.js_shell)
+
+ if test_count == 0:
+ print('no tests selected')
+ return 1
+
+ test_dir = dirname(abspath(__file__))
+
+ if options.debug:
+ if test_count > 1:
+ print('Multiple tests match command line arguments,'
+ ' debugger can only run one')
+ for tc in test_gen:
+ print(' {}'.format(tc.path))
+ return 2
+
+ cmd = test_gen.next().get_command(prefix)
+ if options.show_cmd:
+ print(list2cmdline(cmd))
+ with changedir(test_dir), change_env(test_environment):
+ call(cmd)
+ return 0
+
+ with changedir(test_dir), change_env(test_environment):
+ results = ResultsSink(options, test_count)
+ try:
+ for out in run_all_tests(test_gen, prefix, results.pb, options):
+ results.push(out)
+ results.finish(True)
+ except KeyboardInterrupt:
+ results.finish(False)
+
+ return 0 if results.all_passed() else 1
+
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())