summaryrefslogtreecommitdiffstats
path: root/testing/runcppunittests.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/runcppunittests.py')
-rwxr-xr-xtesting/runcppunittests.py271
1 files changed, 271 insertions, 0 deletions
diff --git a/testing/runcppunittests.py b/testing/runcppunittests.py
new file mode 100755
index 000000000..d8b79f68f
--- /dev/null
+++ b/testing/runcppunittests.py
@@ -0,0 +1,271 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import with_statement
+import sys, os, tempfile, shutil
+from optparse import OptionParser
+import manifestparser
+import mozprocess
+import mozinfo
+import mozcrash
+import mozfile
+import mozlog
+from contextlib import contextmanager
+from subprocess import PIPE
+
+SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+class CPPUnitTests(object):
+ # Time (seconds) to wait for test process to complete
+ TEST_PROC_TIMEOUT = 900
+ # Time (seconds) in which process will be killed if it produces no output.
+ TEST_PROC_NO_OUTPUT_TIMEOUT = 300
+
+ def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+ timeout_factor=1):
+ """
+ Run a single C++ unit test program.
+
+ Arguments:
+ * prog: The path to the test program to run.
+ * env: The environment to use for running the program.
+ * symbols_path: A path to a directory containing Breakpad-formatted
+ symbol files for producing stack traces on crash.
+ * timeout_factor: An optional test-specific timeout multiplier.
+
+ Return True if the program exits with a zero status, False otherwise.
+ """
+ basename = os.path.basename(prog)
+ self.log.test_start(basename)
+ with mozfile.TemporaryDirectory() as tempdir:
+ if interactive:
+ # For tests run locally, via mach, print output directly
+ proc = mozprocess.ProcessHandler([prog],
+ cwd=tempdir,
+ env=env,
+ storeOutput=False)
+ else:
+ proc = mozprocess.ProcessHandler([prog],
+ cwd=tempdir,
+ env=env,
+ storeOutput=True,
+ processOutputLine=lambda _: None)
+ #TODO: After bug 811320 is fixed, don't let .run() kill the process,
+ # instead use a timeout in .wait() and then kill to get a stack.
+ test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
+ proc.run(timeout=test_timeout,
+ outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
+ proc.wait()
+ if proc.output:
+ output = "\n%s" % "\n".join(proc.output)
+ self.log.process_output(proc.pid, output, command=[prog])
+ if proc.timedOut:
+ message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
+ self.log.test_end(basename, status='TIMEOUT', expected='PASS',
+ message=message)
+ return False
+ if mozcrash.check_for_crashes(tempdir, symbols_path,
+ test_name=basename):
+ self.log.test_end(basename, status='CRASH', expected='PASS')
+ return False
+ result = proc.proc.returncode == 0
+ if not result:
+ self.log.test_end(basename, status='FAIL', expected='PASS',
+ message=("test failed with return code %d" %
+ proc.proc.returncode))
+ else:
+ self.log.test_end(basename, status='PASS', expected='PASS')
+ return result
+
+ def build_core_environment(self, env = {}):
+ """
+ Add environment variables likely to be used across all platforms, including remote systems.
+ """
+ env["MOZILLA_FIVE_HOME"] = self.xre_path
+ env["MOZ_XRE_DIR"] = self.xre_path
+ #TODO: switch this to just abort once all C++ unit tests have
+ # been fixed to enable crash reporting
+ env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
+ env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+ env["MOZ_CRASHREPORTER"] = "1"
+ return env
+
+ def build_environment(self):
+ """
+ Create and return a dictionary of all the appropriate env variables and values.
+ On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
+ """
+ if not os.path.isdir(self.xre_path):
+ raise Exception("xre_path does not exist: %s", self.xre_path)
+ env = dict(os.environ)
+ env = self.build_core_environment(env)
+ pathvar = ""
+ libpath = self.xre_path
+ if mozinfo.os == "linux":
+ pathvar = "LD_LIBRARY_PATH"
+ elif mozinfo.os == "mac":
+ applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
+ if os.path.exists(applibpath):
+ # Set the library load path to Contents/MacOS if we're run from
+ # the app bundle.
+ libpath = applibpath
+ pathvar = "DYLD_LIBRARY_PATH"
+ elif mozinfo.os == "win":
+ pathvar = "PATH"
+ if pathvar:
+ if pathvar in env:
+ env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
+ else:
+ env[pathvar] = libpath
+
+ if mozinfo.info["asan"]:
+ # Use llvm-symbolizer for ASan if available/required
+ llvmsym = os.path.join(self.xre_path, "llvm-symbolizer")
+ if os.path.isfile(llvmsym):
+ env["ASAN_SYMBOLIZER_PATH"] = llvmsym
+ self.log.info("ASan using symbolizer at %s" % llvmsym)
+ else:
+ self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
+
+ # media/mtransport tests statically link in NSS, which
+ # causes ODR violations. See bug 1215679.
+ assert not 'ASAN_OPTIONS' in env
+ env['ASAN_OPTIONS'] = 'detect_leaks=0:detect_odr_violation=0'
+
+ return env
+
+ def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
+ """
+ Run a set of C++ unit test programs.
+
+ Arguments:
+ * programs: An iterable containing (test path, test timeout factor) tuples
+ * xre_path: A path to a directory containing a XUL Runtime Environment.
+ * symbols_path: A path to a directory containing Breakpad-formatted
+ symbol files for producing stack traces on crash.
+
+ Returns True if all test programs exited with a zero status, False
+ otherwise.
+ """
+ self.xre_path = xre_path
+ self.log = mozlog.get_default_logger()
+ self.log.suite_start(programs)
+ env = self.build_environment()
+ pass_count = 0
+ fail_count = 0
+ for prog in programs:
+ test_path = prog[0]
+ timeout_factor = prog[1]
+ single_result = self.run_one_test(test_path, env, symbols_path,
+ interactive, timeout_factor)
+ if single_result:
+ pass_count += 1
+ else:
+ fail_count += 1
+ self.log.suite_end()
+
+ # Mozharness-parseable summary formatting.
+ self.log.info("Result summary:")
+ self.log.info("cppunittests INFO | Passed: %d" % pass_count)
+ self.log.info("cppunittests INFO | Failed: %d" % fail_count)
+ return fail_count == 0
+
+class CPPUnittestOptions(OptionParser):
+ def __init__(self):
+ OptionParser.__init__(self)
+ self.add_option("--xre-path",
+ action = "store", type = "string", dest = "xre_path",
+ default = None,
+ help = "absolute path to directory containing XRE (probably xulrunner)")
+ self.add_option("--symbols-path",
+ action = "store", type = "string", dest = "symbols_path",
+ default = None,
+ help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+ self.add_option("--manifest-path",
+ action = "store", type = "string", dest = "manifest_path",
+ default = None,
+ help = "path to test manifest, if different from the path to test binaries")
+
+def extract_unittests_from_args(args, environ, manifest_path):
+ """Extract unittests from args, expanding directories as needed"""
+ mp = manifestparser.TestManifest(strict=True)
+ tests = []
+ binary_path = None
+
+ if manifest_path:
+ mp.read(manifest_path)
+ binary_path = os.path.abspath(args[0])
+ else:
+ for p in args:
+ if os.path.isdir(p):
+ try:
+ mp.read(os.path.join(p, 'cppunittest.ini'))
+ except IOError:
+ files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
+ tests.extend((f, 1) for f in files
+ if os.access(f, os.R_OK | os.X_OK))
+ else:
+ tests.append((os.path.abspath(p), 1))
+
+ # we skip the existence check here because not all tests are built
+ # for all platforms (and it will fail on Windows anyway)
+ active_tests = mp.active_tests(exists=False, disabled=False, **environ)
+ suffix = '.exe' if mozinfo.isWin else ''
+ if binary_path:
+ tests.extend([(os.path.join(binary_path, test['relpath'] + suffix), int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
+ else:
+ tests.extend([(test['path'] + suffix, int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
+
+ # skip non-existing tests
+ tests = [test for test in tests if os.path.isfile(test[0])]
+
+ return tests
+
+def update_mozinfo():
+ """walk up directories to find mozinfo.json update the info"""
+ path = SCRIPT_DIR
+ dirs = set()
+ while path != os.path.expanduser('~'):
+ if path in dirs:
+ break
+ dirs.add(path)
+ path = os.path.split(path)[0]
+ mozinfo.find_and_update_from_json(*dirs)
+
+def run_test_harness(options, args):
+ update_mozinfo()
+ progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
+ options.xre_path = os.path.abspath(options.xre_path)
+ tester = CPPUnitTests()
+ result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+
+ return result
+
+def main():
+ parser = CPPUnittestOptions()
+ mozlog.commandline.add_logging_group(parser)
+ options, args = parser.parse_args()
+ if not args:
+ print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
+ sys.exit(1)
+ if not options.xre_path:
+ print >>sys.stderr, """Error: --xre-path is required"""
+ sys.exit(1)
+ if options.manifest_path and len(args) > 1:
+ print >>sys.stderr, "Error: multiple arguments not supported with --test-manifest"
+ sys.exit(1)
+ log = mozlog.commandline.setup_logging("cppunittests", options,
+ {"tbpl": sys.stdout})
+ try:
+ result = run_test_harness(options, args)
+ except Exception as e:
+ log.error(str(e))
+ result = False
+
+ sys.exit(0 if result else 1)
+
+if __name__ == '__main__':
+ main()