summaryrefslogtreecommitdiffstats
path: root/layout/tools/reftest/reftestcommandline.py
diff options
context:
space:
mode:
Diffstat (limited to 'layout/tools/reftest/reftestcommandline.py')
-rw-r--r--layout/tools/reftest/reftestcommandline.py749
1 files changed, 749 insertions, 0 deletions
diff --git a/layout/tools/reftest/reftestcommandline.py b/layout/tools/reftest/reftestcommandline.py
new file mode 100644
index 000000000..da76fbd9a
--- /dev/null
+++ b/layout/tools/reftest/reftestcommandline.py
@@ -0,0 +1,749 @@
+import argparse
+import os
+import sys
+from collections import OrderedDict
+from urlparse import urlparse
+
+import mozlog
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class ReftestArgumentsParser(argparse.ArgumentParser):
+ def __init__(self, **kwargs):
+ super(ReftestArgumentsParser, self).__init__(**kwargs)
+
+ # Try to import a MozbuildObject. Success indicates that we are
+ # running from a source tree. This allows some defaults to be set
+ # from the source tree.
+ try:
+ from mozbuild.base import MozbuildObject
+ self.build_obj = MozbuildObject.from_environment(cwd=here)
+ except ImportError:
+ self.build_obj = None
+
+ self.add_argument("--xre-path",
+ action="store",
+ type=str,
+ dest="xrePath",
+ # individual scripts will set a sane default
+ default=None,
+ help="absolute path to directory containing XRE (probably xulrunner)")
+
+ self.add_argument("--symbols-path",
+ action="store",
+ type=str,
+ dest="symbolsPath",
+ default=None,
+ help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+
+ self.add_argument("--debugger",
+ action="store",
+ dest="debugger",
+ help="use the given debugger to launch the application")
+
+ self.add_argument("--debugger-args",
+ action="store",
+ dest="debuggerArgs",
+ help="pass the given args to the debugger _before_ "
+ "the application on the command line")
+
+ self.add_argument("--debugger-interactive",
+ action="store_true",
+ dest="debuggerInteractive",
+ help="prevents the test harness from redirecting "
+ "stdout and stderr for interactive debuggers")
+
+ self.add_argument("--appname",
+ action="store",
+ type=str,
+ dest="app",
+ default=None,
+ help="absolute path to application, overriding default")
+
+ self.add_argument("--extra-profile-file",
+ action="append",
+ dest="extraProfileFiles",
+ default=[],
+ help="copy specified files/dirs to testing profile")
+
+ self.add_argument("--timeout",
+ action="store",
+ dest="timeout",
+ type=int,
+ default=5 * 60, # 5 minutes per bug 479518
+ help="reftest will timeout in specified number of seconds. [default %(default)s].")
+
+ self.add_argument("--leak-threshold",
+ action="store",
+ type=int,
+ dest="defaultLeakThreshold",
+ default=0,
+ help="fail if the number of bytes leaked in default "
+ "processes through refcounted objects (or bytes "
+ "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
+ "is greater than the given number")
+
+ self.add_argument("--utility-path",
+ action="store",
+ type=str,
+ dest="utilityPath",
+ default=self.build_obj.bindir if self.build_obj else None,
+ help="absolute path to directory containing utility "
+ "programs (xpcshell, ssltunnel, certutil)")
+
+ self.add_argument("--total-chunks",
+ type=int,
+ dest="totalChunks",
+ help="how many chunks to split the tests up into")
+
+ self.add_argument("--this-chunk",
+ type=int,
+ dest="thisChunk",
+ help="which chunk to run between 1 and --total-chunks")
+
+ self.add_argument("--log-file",
+ action="store",
+ type=str,
+ dest="logFile",
+ default=None,
+ help="file to log output to in addition to stdout")
+
+ self.add_argument("--skip-slow-tests",
+ dest="skipSlowTests",
+ action="store_true",
+ default=False,
+ help="skip tests marked as slow when running")
+
+ self.add_argument("--ignore-window-size",
+ dest="ignoreWindowSize",
+ action="store_true",
+ default=False,
+ help="ignore the window size, which may cause spurious failures and passes")
+
+ self.add_argument("--install-extension",
+ action="append",
+ dest="extensionsToInstall",
+ default=[],
+ help="install the specified extension in the testing profile. "
+ "The extension file's name should be <id>.xpi where <id> is "
+ "the extension's id as indicated in its install.rdf. "
+ "An optional path can be specified too.")
+
+ self.add_argument("--marionette",
+ default=None,
+ help="host:port to use when connecting to Marionette")
+
+ self.add_argument("--marionette-port-timeout",
+ default=None,
+ help=argparse.SUPPRESS)
+
+ self.add_argument("--marionette-socket-timeout",
+ default=None,
+ help=argparse.SUPPRESS)
+
+ self.add_argument("--marionette-startup-timeout",
+ default=None,
+ help=argparse.SUPPRESS)
+
+ self.add_argument("--setenv",
+ action="append",
+ type=str,
+ default=[],
+ dest="environment",
+ metavar="NAME=VALUE",
+ help="sets the given variable in the application's "
+ "environment")
+
+ self.add_argument("--filter",
+ action="store",
+ type=str,
+ dest="filter",
+ help="specifies a regular expression (as could be passed to the JS "
+ "RegExp constructor) to test against URLs in the reftest manifest; "
+ "only test items that have a matching test URL will be run.")
+
+ self.add_argument("--shuffle",
+ action="store_true",
+ default=False,
+ dest="shuffle",
+ help="run reftests in random order")
+
+ self.add_argument("--run-until-failure",
+ action="store_true",
+ default=False,
+ dest="runUntilFailure",
+ help="stop running on the first failure. Useful for RR recordings.")
+
+ self.add_argument("--repeat",
+ action="store",
+ type=int,
+ default=0,
+ dest="repeat",
+ help="number of times the select test(s) will be executed. Useful for "
+ "finding intermittent failures.")
+
+ self.add_argument("--focus-filter-mode",
+ action="store",
+ type=str,
+ dest="focusFilterMode",
+ default="all",
+ help="filters tests to run by whether they require focus. "
+ "Valid values are `all', `needs-focus', or `non-needs-focus'. "
+ "Defaults to `all'.")
+
+ self.add_argument("--disable-e10s",
+ action="store_false",
+ default=True,
+ dest="e10s",
+ help="disables content processes")
+
+ self.add_argument("--setpref",
+ action="append",
+ type=str,
+ default=[],
+ dest="extraPrefs",
+ metavar="PREF=VALUE",
+ help="defines an extra user preference")
+
+ self.add_argument("--reftest-extension-path",
+ action="store",
+ dest="reftestExtensionPath",
+ help="Path to the reftest extension")
+
+ self.add_argument("--special-powers-extension-path",
+ action="store",
+ dest="specialPowersExtensionPath",
+ help="Path to the special powers extension")
+
+ self.add_argument("--suite",
+ choices=["reftest", "crashtest", "jstestbrowser"],
+ default=None,
+ help=argparse.SUPPRESS)
+
+ self.add_argument("--cleanup-crashes",
+ action = "store_true",
+ dest = "cleanupCrashes",
+ default = False,
+ help = "Delete pending crash reports before running tests.")
+
+ self.add_argument("tests",
+ metavar="TEST_PATH",
+ nargs="*",
+ help="Path to test file, manifest file, or directory containing tests")
+
+ mozlog.commandline.add_logging_group(self)
+
+ def get_ip(self):
+ import moznetwork
+ if os.name != "nt":
+ return moznetwork.get_ip()
+ else:
+ self.error(
+ "ERROR: you must specify a --remote-webserver=<ip address>\n")
+
+ def set_default_suite(self, options):
+ manifests = OrderedDict([("reftest.list", "reftest"),
+ ("crashtests.list", "crashtest"),
+ ("jstests.list", "jstestbrowser")])
+
+ for test_path in options.tests:
+ file_name = os.path.basename(test_path)
+ if file_name in manifests:
+ options.suite = manifests[file_name]
+ return
+
+ for test_path in options.tests:
+ for manifest_file, suite in manifests.iteritems():
+ if os.path.exists(os.path.join(test_path, manifest_file)):
+ options.suite = suite
+ return
+
+ self.error("Failed to determine test suite; supply --suite to set this explicitly")
+
+ def validate(self, options, reftest):
+ if not options.tests:
+ # Can't just set this in the argument parser because mach will set a default
+ self.error("Must supply at least one path to a manifest file, test directory, or test file to run.")
+
+ if options.suite is None:
+ self.set_default_suite(options)
+
+ if options.totalChunks is not None and options.thisChunk is None:
+ self.error(
+ "thisChunk must be specified when totalChunks is specified")
+
+ if options.totalChunks:
+ if not 1 <= options.thisChunk <= options.totalChunks:
+ self.error("thisChunk must be between 1 and totalChunks")
+
+ if options.logFile:
+ options.logFile = reftest.getFullPath(options.logFile)
+
+ if options.xrePath is not None:
+ if not os.access(options.xrePath, os.F_OK):
+ self.error("--xre-path '%s' not found" % options.xrePath)
+ if not os.path.isdir(options.xrePath):
+ self.error("--xre-path '%s' is not a directory" %
+ options.xrePath)
+ options.xrePath = reftest.getFullPath(options.xrePath)
+
+ if options.reftestExtensionPath is None:
+ if self.build_obj is not None:
+ reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
+ "reftest", "reftest")
+ else:
+ reftestExtensionPath = os.path.join(here, "reftest")
+ options.reftestExtensionPath = os.path.normpath(reftestExtensionPath)
+
+ if (options.specialPowersExtensionPath is None and
+ options.suite in ["crashtest", "jstestbrowser"]):
+ if self.build_obj is not None:
+ specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
+ "reftest", "specialpowers")
+ else:
+ specialPowersExtensionPath = os.path.join(here, "specialpowers")
+ options.specialPowersExtensionPath = os.path.normpath(specialPowersExtensionPath)
+
+ options.leakThresholds = {
+ "default": options.defaultLeakThreshold,
+ "tab": 5000, # See dependencies of bug 1051230.
+ }
+
+
+class DesktopArgumentsParser(ReftestArgumentsParser):
+ def __init__(self, **kwargs):
+ super(DesktopArgumentsParser, self).__init__(**kwargs)
+
+ self.add_argument("--run-tests-in-parallel",
+ action="store_true",
+ default=False,
+ dest="runTestsInParallel",
+ help="run tests in parallel if possible")
+
+ def _prefs_gpu(self):
+ if mozinfo.os != "win":
+ return ["layers.acceleration.force-enabled=true"]
+ return []
+
+ def validate(self, options, reftest):
+ super(DesktopArgumentsParser, self).validate(options, reftest)
+
+ if options.runTestsInParallel:
+ if options.logFile is not None:
+ self.error("cannot specify logfile with parallel tests")
+ if options.totalChunks is not None or options.thisChunk is not None:
+ self.error(
+ "cannot specify thisChunk or totalChunks with parallel tests")
+ if options.focusFilterMode != "all":
+ self.error("cannot specify focusFilterMode with parallel tests")
+ if options.debugger is not None:
+ self.error("cannot specify a debugger with parallel tests")
+
+ if options.debugger:
+ # valgrind and some debuggers may cause Gecko to start slowly. Make sure
+ # marionette waits long enough to connect.
+ options.marionette_port_timeout = 900
+ options.marionette_socket_timeout = 540
+
+ if not options.tests:
+ self.error("No test files specified.")
+
+ if options.app is None:
+ bin_dir = (self.build_obj.get_binary_path() if
+ self.build_obj and self.build_obj.substs[
+ 'MOZ_BUILD_APP'] != 'mobile/android'
+ else None)
+
+ if bin_dir:
+ options.app = bin_dir
+
+ if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
+ options.symbolsPath = reftest.getFullPath(options.symbolsPath)
+
+ options.utilityPath = reftest.getFullPath(options.utilityPath)
+
+
+class B2GArgumentParser(ReftestArgumentsParser):
+ def __init__(self, **kwargs):
+ super(B2GArgumentParser, self).__init__(**kwargs)
+
+ self.add_argument("--browser-arg",
+ action="store",
+ type=str,
+ dest="browser_arg",
+ help="Optional command-line arg to pass to the browser")
+
+ self.add_argument("--b2gpath",
+ action="store",
+ type=str,
+ dest="b2gPath",
+ help="path to B2G repo or qemu dir")
+
+ self.add_argument("--emulator",
+ action="store",
+ type=str,
+ dest="emulator",
+ help="Architecture of emulator to use: x86 or arm")
+
+ self.add_argument("--emulator-res",
+ action="store",
+ type=str,
+ dest="emulator_res",
+ help="Emulator resolution of the format '<width>x<height>'")
+
+ self.add_argument("--no-window",
+ action="store_true",
+ dest="noWindow",
+ default=False,
+ help="Pass --no-window to the emulator")
+
+ self.add_argument("--adbpath",
+ action="store",
+ type=str,
+ dest="adb_path",
+ default="adb",
+ help="path to adb")
+
+ self.add_argument("--deviceIP",
+ action="store",
+ type=str,
+ dest="deviceIP",
+ help="ip address of remote device to test")
+
+ self.add_argument("--devicePort",
+ action="store",
+ type=str,
+ dest="devicePort",
+ default="20701",
+ help="port of remote device to test")
+
+ self.add_argument("--remote-logfile",
+ action="store",
+ type=str,
+ dest="remoteLogFile",
+ help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
+
+ self.add_argument("--remote-webserver",
+ action="store",
+ type=str,
+ dest="remoteWebServer",
+ help="ip address where the remote web server is hosted at")
+
+ self.add_argument("--http-port",
+ action="store",
+ type=str,
+ dest="httpPort",
+ help="ip address where the remote web server is hosted at")
+
+ self.add_argument("--ssl-port",
+ action="store",
+ type=str,
+ dest="sslPort",
+ help="ip address where the remote web server is hosted at")
+
+ self.add_argument("--pidfile",
+ action="store",
+ type=str,
+ dest="pidFile",
+ default="",
+ help="name of the pidfile to generate")
+
+ self.add_argument("--gecko-path",
+ action="store",
+ type=str,
+ dest="geckoPath",
+ help="the path to a gecko distribution that should "
+ "be installed on the emulator prior to test")
+
+ self.add_argument("--logdir",
+ action="store",
+ type=str,
+ dest="logdir",
+ help="directory to store log files")
+
+ self.add_argument('--busybox',
+ action='store',
+ type=str,
+ dest='busybox',
+ help="Path to busybox binary to install on device")
+
+ self.add_argument("--httpd-path",
+ action="store",
+ type=str,
+ dest="httpdPath",
+ help="path to the httpd.js file")
+
+ self.add_argument("--profile",
+ action="store",
+ type=str,
+ dest="profile",
+ help="for mulet testing, the path to the "
+ "gaia profile to use")
+
+ self.add_argument("--mulet",
+ action="store_true",
+ dest="mulet",
+ default=False,
+ help="Run the tests on a B2G desktop build")
+
+ self.set_defaults(remoteTestRoot=None,
+ logFile="reftest.log",
+ autorun=True,
+ closeWhenDone=True,
+ testPath="")
+
+ def validate_remote(self, options, automation):
+ if not options.app:
+ options.app = automation.DEFAULT_APP
+
+ if not options.remoteTestRoot:
+ options.remoteTestRoot = automation._devicemanager.deviceRoot + \
+ "/reftest"
+
+ options.remoteProfile = options.remoteTestRoot + "/profile"
+
+ productRoot = options.remoteTestRoot + "/" + automation._product
+ if options.utilityPath is None:
+ options.utilityPath = productRoot + "/bin"
+
+ if not options.httpPort:
+ options.httpPort = automation.DEFAULT_HTTP_PORT
+
+ if not options.sslPort:
+ options.sslPort = automation.DEFAULT_SSL_PORT
+
+ if options.remoteWebServer is None:
+ options.remoteWebServer = self.get_ip()
+
+ options.webServer = options.remoteWebServer
+
+ if options.geckoPath and not options.emulator:
+ self.error(
+ "You must specify --emulator if you specify --gecko-path")
+
+ if options.logdir and not options.emulator:
+ self.error("You must specify --emulator if you specify --logdir")
+
+ if options.remoteLogFile is None:
+ options.remoteLogFile = "reftest.log"
+
+ options.localLogName = options.remoteLogFile
+ options.remoteLogFile = options.remoteTestRoot + \
+ '/' + options.remoteLogFile
+
+ # Ensure that the options.logfile (which the base class uses) is set to
+ # the remote setting when running remote. Also, if the user set the
+ # log file name there, use that instead of reusing the remotelogfile as
+ # above.
+ if (options.logFile):
+ # If the user specified a local logfile name use that
+ options.localLogName = options.logFile
+ options.logFile = options.remoteLogFile
+
+ # Only reset the xrePath if it wasn't provided
+ if options.xrePath is None:
+ options.xrePath = options.utilityPath
+ options.xrePath = os.path.abspath(options.xrePath)
+
+ if options.pidFile != "":
+ f = open(options.pidFile, 'w')
+ f.write("%s" % os.getpid())
+ f.close()
+
+ # httpd-path is specified by standard makefile targets and may be specified
+ # on the command line to select a particular version of httpd.js. If not
+ # specified, try to select the one from from the xre bundle, as
+ # required in bug 882932.
+ if not options.httpdPath:
+ options.httpdPath = os.path.join(options.xrePath, "components")
+
+ return options
+
+
+class RemoteArgumentsParser(ReftestArgumentsParser):
+ def __init__(self, **kwargs):
+ super(RemoteArgumentsParser, self).__init__()
+
+ # app, xrePath and utilityPath variables are set in main function
+ self.set_defaults(logFile="reftest.log",
+ app="",
+ xrePath="",
+ utilityPath="",
+ localLogName=None)
+
+ self.add_argument("--remote-app-path",
+ action="store",
+ type=str,
+ dest="remoteAppPath",
+ help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
+
+ self.add_argument("--adbpath",
+ action="store",
+ type=str,
+ dest="adb_path",
+ default="adb",
+ help="path to adb")
+
+ self.add_argument("--deviceIP",
+ action="store",
+ type=str,
+ dest="deviceIP",
+ help="ip address of remote device to test")
+
+ self.add_argument("--deviceSerial",
+ action="store",
+ type=str,
+ dest="deviceSerial",
+ help="adb serial number of remote device to test")
+
+ self.add_argument("--devicePort",
+ action="store",
+ type=str,
+ default="20701",
+ dest="devicePort",
+ help="port of remote device to test")
+
+ self.add_argument("--remote-product-name",
+ action="store",
+ type=str,
+ dest="remoteProductName",
+ default="fennec",
+ help="Name of product to test - either fennec or firefox, defaults to fennec")
+
+ self.add_argument("--remote-webserver",
+ action="store",
+ type=str,
+ dest="remoteWebServer",
+ help="IP Address of the webserver hosting the reftest content")
+
+ self.add_argument("--http-port",
+ action="store",
+ type=str,
+ dest="httpPort",
+ help="port of the web server for http traffic")
+
+ self.add_argument("--ssl-port",
+ action="store",
+ type=str,
+ dest="sslPort",
+ help="Port for https traffic to the web server")
+
+ self.add_argument("--remote-logfile",
+ action="store",
+ type=str,
+ dest="remoteLogFile",
+ default="reftest.log",
+ help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
+
+ self.add_argument("--pidfile",
+ action="store",
+ type=str,
+ dest="pidFile",
+ default="",
+ help="name of the pidfile to generate")
+
+ self.add_argument("--dm_trans",
+ action="store",
+ type=str,
+ dest="dm_trans",
+ default="sut",
+ help="the transport to use to communicate with device: [adb|sut]; default=sut")
+
+ self.add_argument("--remoteTestRoot",
+ action="store",
+ type=str,
+ dest="remoteTestRoot",
+ help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
+
+ self.add_argument("--httpd-path",
+ action="store",
+ type=str,
+ dest="httpdPath",
+ help="path to the httpd.js file")
+
+ self.add_argument("--no-device-info",
+ action="store_false",
+ dest="printDeviceInfo",
+ default=True,
+ help="do not display verbose diagnostics about the remote device")
+
+ def validate_remote(self, options, automation):
+ # Ensure our defaults are set properly for everything we can infer
+ if not options.remoteTestRoot:
+ options.remoteTestRoot = automation._devicemanager.deviceRoot + \
+ '/reftest'
+ options.remoteProfile = options.remoteTestRoot + "/profile"
+
+ if options.remoteWebServer is None:
+ options.remoteWebServer = self.get_ip()
+
+ # Verify that our remotewebserver is set properly
+ if options.remoteWebServer == '127.0.0.1':
+ self.error("ERROR: Either you specified the loopback for the remote webserver or ",
+ "your local IP cannot be detected. Please provide the local ip in --remote-webserver")
+
+ if not options.httpPort:
+ options.httpPort = automation.DEFAULT_HTTP_PORT
+
+ if not options.sslPort:
+ options.sslPort = automation.DEFAULT_SSL_PORT
+
+ # One of remoteAppPath (relative path to application) or the app (executable) must be
+ # set, but not both. If both are set, we destroy the user's selection for app
+ # so instead of silently destroying a user specificied setting, we
+ # error.
+ if options.remoteAppPath and options.app:
+ self.error(
+ "ERROR: You cannot specify both the remoteAppPath and the app")
+ elif options.remoteAppPath:
+ options.app = options.remoteTestRoot + "/" + options.remoteAppPath
+ elif options.app is None:
+ # Neither remoteAppPath nor app are set -- error
+ self.error("ERROR: You must specify either appPath or app")
+
+ if options.xrePath is None:
+ self.error(
+ "ERROR: You must specify the path to the controller xre directory")
+ else:
+ # Ensure xrepath is a full path
+ options.xrePath = os.path.abspath(options.xrePath)
+
+ options.localLogName = options.remoteLogFile
+ options.remoteLogFile = options.remoteTestRoot + \
+ '/' + options.remoteLogFile
+
+ # Ensure that the options.logfile (which the base class uses) is set to
+ # the remote setting when running remote. Also, if the user set the
+ # log file name there, use that instead of reusing the remotelogfile as
+ # above.
+ if options.logFile:
+ # If the user specified a local logfile name use that
+ options.localLogName = options.logFile
+
+ options.logFile = options.remoteLogFile
+
+ if options.pidFile != "":
+ with open(options.pidFile, 'w') as f:
+ f.write(str(os.getpid()))
+
+ # httpd-path is specified by standard makefile targets and may be specified
+ # on the command line to select a particular version of httpd.js. If not
+ # specified, try to select the one from hostutils.zip, as required in
+ # bug 882932.
+ if not options.httpdPath:
+ options.httpdPath = os.path.join(options.utilityPath, "components")
+
+ if not options.ignoreWindowSize:
+ parts = automation._devicemanager.getInfo(
+ 'screen')['screen'][0].split()
+ width = int(parts[0].split(':')[1])
+ height = int(parts[1].split(':')[1])
+ if (width < 1366 or height < 1050):
+ self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (
+ width, height))
+
+ # Disable e10s by default on Android because we don't run Android
+ # e10s jobs anywhere yet.
+ options.e10s = False
+ return options
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 15:04:38 +0000