import argparse import os import sys from collections import OrderedDict from urlparse import urlparse import mozlog here = os.path.abspath(os.path.dirname(__file__)) class ReftestArgumentsParser(argparse.ArgumentParser): def __init__(self, **kwargs): super(ReftestArgumentsParser, self).__init__(**kwargs) # Try to import a MozbuildObject. Success indicates that we are # running from a source tree. This allows some defaults to be set # from the source tree. try: from mozbuild.base import MozbuildObject self.build_obj = MozbuildObject.from_environment(cwd=here) except ImportError: self.build_obj = None self.add_argument("--xre-path", action="store", type=str, dest="xrePath", # individual scripts will set a sane default default=None, help="absolute path to directory containing XRE (probably xulrunner)") self.add_argument("--symbols-path", action="store", type=str, dest="symbolsPath", default=None, help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols") self.add_argument("--debugger", action="store", dest="debugger", help="use the given debugger to launch the application") self.add_argument("--debugger-args", action="store", dest="debuggerArgs", help="pass the given args to the debugger _before_ " "the application on the command line") self.add_argument("--debugger-interactive", action="store_true", dest="debuggerInteractive", help="prevents the test harness from redirecting " "stdout and stderr for interactive debuggers") self.add_argument("--appname", action="store", type=str, dest="app", default=None, help="absolute path to application, overriding default") self.add_argument("--extra-profile-file", action="append", dest="extraProfileFiles", default=[], help="copy specified files/dirs to testing profile") self.add_argument("--timeout", action="store", dest="timeout", type=int, default=5 * 60, # 5 minutes per bug 479518 help="reftest will timeout in specified number of seconds. [default %(default)s].") self.add_argument("--leak-threshold", action="store", type=int, dest="defaultLeakThreshold", default=0, help="fail if the number of bytes leaked in default " "processes through refcounted objects (or bytes " "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " "is greater than the given number") self.add_argument("--utility-path", action="store", type=str, dest="utilityPath", default=self.build_obj.bindir if self.build_obj else None, help="absolute path to directory containing utility " "programs (xpcshell, ssltunnel, certutil)") self.add_argument("--total-chunks", type=int, dest="totalChunks", help="how many chunks to split the tests up into") self.add_argument("--this-chunk", type=int, dest="thisChunk", help="which chunk to run between 1 and --total-chunks") self.add_argument("--log-file", action="store", type=str, dest="logFile", default=None, help="file to log output to in addition to stdout") self.add_argument("--skip-slow-tests", dest="skipSlowTests", action="store_true", default=False, help="skip tests marked as slow when running") self.add_argument("--ignore-window-size", dest="ignoreWindowSize", action="store_true", default=False, help="ignore the window size, which may cause spurious failures and passes") self.add_argument("--install-extension", action="append", dest="extensionsToInstall", default=[], help="install the specified extension in the testing profile. " "The extension file's name should be .xpi where is " "the extension's id as indicated in its install.rdf. " "An optional path can be specified too.") self.add_argument("--marionette", default=None, help="host:port to use when connecting to Marionette") self.add_argument("--marionette-port-timeout", default=None, help=argparse.SUPPRESS) self.add_argument("--marionette-socket-timeout", default=None, help=argparse.SUPPRESS) self.add_argument("--marionette-startup-timeout", default=None, help=argparse.SUPPRESS) self.add_argument("--setenv", action="append", type=str, default=[], dest="environment", metavar="NAME=VALUE", help="sets the given variable in the application's " "environment") self.add_argument("--filter", action="store", type=str, dest="filter", help="specifies a regular expression (as could be passed to the JS " "RegExp constructor) to test against URLs in the reftest manifest; " "only test items that have a matching test URL will be run.") self.add_argument("--shuffle", action="store_true", default=False, dest="shuffle", help="run reftests in random order") self.add_argument("--run-until-failure", action="store_true", default=False, dest="runUntilFailure", help="stop running on the first failure. Useful for RR recordings.") self.add_argument("--repeat", action="store", type=int, default=0, dest="repeat", help="number of times the select test(s) will be executed. Useful for " "finding intermittent failures.") self.add_argument("--focus-filter-mode", action="store", type=str, dest="focusFilterMode", default="all", help="filters tests to run by whether they require focus. " "Valid values are `all', `needs-focus', or `non-needs-focus'. " "Defaults to `all'.") self.add_argument("--disable-e10s", action="store_false", default=True, dest="e10s", help="disables content processes") self.add_argument("--setpref", action="append", type=str, default=[], dest="extraPrefs", metavar="PREF=VALUE", help="defines an extra user preference") self.add_argument("--reftest-extension-path", action="store", dest="reftestExtensionPath", help="Path to the reftest extension") self.add_argument("--special-powers-extension-path", action="store", dest="specialPowersExtensionPath", help="Path to the special powers extension") self.add_argument("--suite", choices=["reftest", "crashtest", "jstestbrowser"], default=None, help=argparse.SUPPRESS) self.add_argument("--cleanup-crashes", action = "store_true", dest = "cleanupCrashes", default = False, help = "Delete pending crash reports before running tests.") self.add_argument("tests", metavar="TEST_PATH", nargs="*", help="Path to test file, manifest file, or directory containing tests") mozlog.commandline.add_logging_group(self) def get_ip(self): import moznetwork if os.name != "nt": return moznetwork.get_ip() else: self.error( "ERROR: you must specify a --remote-webserver=\n") def set_default_suite(self, options): manifests = OrderedDict([("reftest.list", "reftest"), ("crashtests.list", "crashtest"), ("jstests.list", "jstestbrowser")]) for test_path in options.tests: file_name = os.path.basename(test_path) if file_name in manifests: options.suite = manifests[file_name] return for test_path in options.tests: for manifest_file, suite in manifests.iteritems(): if os.path.exists(os.path.join(test_path, manifest_file)): options.suite = suite return self.error("Failed to determine test suite; supply --suite to set this explicitly") def validate(self, options, reftest): if not options.tests: # Can't just set this in the argument parser because mach will set a default self.error("Must supply at least one path to a manifest file, test directory, or test file to run.") if options.suite is None: self.set_default_suite(options) if options.totalChunks is not None and options.thisChunk is None: self.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: self.error("thisChunk must be between 1 and totalChunks") if options.logFile: options.logFile = reftest.getFullPath(options.logFile) if options.xrePath is not None: if not os.access(options.xrePath, os.F_OK): self.error("--xre-path '%s' not found" % options.xrePath) if not os.path.isdir(options.xrePath): self.error("--xre-path '%s' is not a directory" % options.xrePath) options.xrePath = reftest.getFullPath(options.xrePath) if options.reftestExtensionPath is None: if self.build_obj is not None: reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests", "reftest", "reftest") else: reftestExtensionPath = os.path.join(here, "reftest") options.reftestExtensionPath = os.path.normpath(reftestExtensionPath) if (options.specialPowersExtensionPath is None and options.suite in ["crashtest", "jstestbrowser"]): if self.build_obj is not None: specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests", "reftest", "specialpowers") else: specialPowersExtensionPath = os.path.join(here, "specialpowers") options.specialPowersExtensionPath = os.path.normpath(specialPowersExtensionPath) options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": 5000, # See dependencies of bug 1051230. } class DesktopArgumentsParser(ReftestArgumentsParser): def __init__(self, **kwargs): super(DesktopArgumentsParser, self).__init__(**kwargs) self.add_argument("--run-tests-in-parallel", action="store_true", default=False, dest="runTestsInParallel", help="run tests in parallel if possible") def _prefs_gpu(self): if mozinfo.os != "win": return ["layers.acceleration.force-enabled=true"] return [] def validate(self, options, reftest): super(DesktopArgumentsParser, self).validate(options, reftest) if options.runTestsInParallel: if options.logFile is not None: self.error("cannot specify logfile with parallel tests") if options.totalChunks is not None or options.thisChunk is not None: self.error( "cannot specify thisChunk or totalChunks with parallel tests") if options.focusFilterMode != "all": self.error("cannot specify focusFilterMode with parallel tests") if options.debugger is not None: self.error("cannot specify a debugger with parallel tests") if options.debugger: # valgrind and some debuggers may cause Gecko to start slowly. Make sure # marionette waits long enough to connect. options.marionette_port_timeout = 900 options.marionette_socket_timeout = 540 if not options.tests: self.error("No test files specified.") if options.app is None: bin_dir = (self.build_obj.get_binary_path() if self.build_obj and self.build_obj.substs[ 'MOZ_BUILD_APP'] != 'mobile/android' else None) if bin_dir: options.app = bin_dir if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: options.symbolsPath = reftest.getFullPath(options.symbolsPath) options.utilityPath = reftest.getFullPath(options.utilityPath) class B2GArgumentParser(ReftestArgumentsParser): def __init__(self, **kwargs): super(B2GArgumentParser, self).__init__(**kwargs) self.add_argument("--browser-arg", action="store", type=str, dest="browser_arg", help="Optional command-line arg to pass to the browser") self.add_argument("--b2gpath", action="store", type=str, dest="b2gPath", help="path to B2G repo or qemu dir") self.add_argument("--emulator", action="store", type=str, dest="emulator", help="Architecture of emulator to use: x86 or arm") self.add_argument("--emulator-res", action="store", type=str, dest="emulator_res", help="Emulator resolution of the format 'x'") self.add_argument("--no-window", action="store_true", dest="noWindow", default=False, help="Pass --no-window to the emulator") self.add_argument("--adbpath", action="store", type=str, dest="adb_path", default="adb", help="path to adb") self.add_argument("--deviceIP", action="store", type=str, dest="deviceIP", help="ip address of remote device to test") self.add_argument("--devicePort", action="store", type=str, dest="devicePort", default="20701", help="port of remote device to test") self.add_argument("--remote-logfile", action="store", type=str, dest="remoteLogFile", help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.") self.add_argument("--remote-webserver", action="store", type=str, dest="remoteWebServer", help="ip address where the remote web server is hosted at") self.add_argument("--http-port", action="store", type=str, dest="httpPort", help="ip address where the remote web server is hosted at") self.add_argument("--ssl-port", action="store", type=str, dest="sslPort", help="ip address where the remote web server is hosted at") self.add_argument("--pidfile", action="store", type=str, dest="pidFile", default="", help="name of the pidfile to generate") self.add_argument("--gecko-path", action="store", type=str, dest="geckoPath", help="the path to a gecko distribution that should " "be installed on the emulator prior to test") self.add_argument("--logdir", action="store", type=str, dest="logdir", help="directory to store log files") self.add_argument('--busybox', action='store', type=str, dest='busybox', help="Path to busybox binary to install on device") self.add_argument("--httpd-path", action="store", type=str, dest="httpdPath", help="path to the httpd.js file") self.add_argument("--profile", action="store", type=str, dest="profile", help="for mulet testing, the path to the " "gaia profile to use") self.add_argument("--mulet", action="store_true", dest="mulet", default=False, help="Run the tests on a B2G desktop build") self.set_defaults(remoteTestRoot=None, logFile="reftest.log", autorun=True, closeWhenDone=True, testPath="") def validate_remote(self, options, automation): if not options.app: options.app = automation.DEFAULT_APP if not options.remoteTestRoot: options.remoteTestRoot = automation._devicemanager.deviceRoot + \ "/reftest" options.remoteProfile = options.remoteTestRoot + "/profile" productRoot = options.remoteTestRoot + "/" + automation._product if options.utilityPath is None: options.utilityPath = productRoot + "/bin" if not options.httpPort: options.httpPort = automation.DEFAULT_HTTP_PORT if not options.sslPort: options.sslPort = automation.DEFAULT_SSL_PORT if options.remoteWebServer is None: options.remoteWebServer = self.get_ip() options.webServer = options.remoteWebServer if options.geckoPath and not options.emulator: self.error( "You must specify --emulator if you specify --gecko-path") if options.logdir and not options.emulator: self.error("You must specify --emulator if you specify --logdir") if options.remoteLogFile is None: options.remoteLogFile = "reftest.log" options.localLogName = options.remoteLogFile options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile # Ensure that the options.logfile (which the base class uses) is set to # the remote setting when running remote. Also, if the user set the # log file name there, use that instead of reusing the remotelogfile as # above. if (options.logFile): # If the user specified a local logfile name use that options.localLogName = options.logFile options.logFile = options.remoteLogFile # Only reset the xrePath if it wasn't provided if options.xrePath is None: options.xrePath = options.utilityPath options.xrePath = os.path.abspath(options.xrePath) if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() # httpd-path is specified by standard makefile targets and may be specified # on the command line to select a particular version of httpd.js. If not # specified, try to select the one from from the xre bundle, as # required in bug 882932. if not options.httpdPath: options.httpdPath = os.path.join(options.xrePath, "components") return options class RemoteArgumentsParser(ReftestArgumentsParser): def __init__(self, **kwargs): super(RemoteArgumentsParser, self).__init__() # app, xrePath and utilityPath variables are set in main function self.set_defaults(logFile="reftest.log", app="", xrePath="", utilityPath="", localLogName=None) self.add_argument("--remote-app-path", action="store", type=str, dest="remoteAppPath", help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.") self.add_argument("--adbpath", action="store", type=str, dest="adb_path", default="adb", help="path to adb") self.add_argument("--deviceIP", action="store", type=str, dest="deviceIP", help="ip address of remote device to test") self.add_argument("--deviceSerial", action="store", type=str, dest="deviceSerial", help="adb serial number of remote device to test") self.add_argument("--devicePort", action="store", type=str, default="20701", dest="devicePort", help="port of remote device to test") self.add_argument("--remote-product-name", action="store", type=str, dest="remoteProductName", default="fennec", help="Name of product to test - either fennec or firefox, defaults to fennec") self.add_argument("--remote-webserver", action="store", type=str, dest="remoteWebServer", help="IP Address of the webserver hosting the reftest content") self.add_argument("--http-port", action="store", type=str, dest="httpPort", help="port of the web server for http traffic") self.add_argument("--ssl-port", action="store", type=str, dest="sslPort", help="Port for https traffic to the web server") self.add_argument("--remote-logfile", action="store", type=str, dest="remoteLogFile", default="reftest.log", help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.") self.add_argument("--pidfile", action="store", type=str, dest="pidFile", default="", help="name of the pidfile to generate") self.add_argument("--dm_trans", action="store", type=str, dest="dm_trans", default="sut", help="the transport to use to communicate with device: [adb|sut]; default=sut") self.add_argument("--remoteTestRoot", action="store", type=str, dest="remoteTestRoot", help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)") self.add_argument("--httpd-path", action="store", type=str, dest="httpdPath", help="path to the httpd.js file") self.add_argument("--no-device-info", action="store_false", dest="printDeviceInfo", default=True, help="do not display verbose diagnostics about the remote device") def validate_remote(self, options, automation): # Ensure our defaults are set properly for everything we can infer if not options.remoteTestRoot: options.remoteTestRoot = automation._devicemanager.deviceRoot + \ '/reftest' options.remoteProfile = options.remoteTestRoot + "/profile" if options.remoteWebServer is None: options.remoteWebServer = self.get_ip() # Verify that our remotewebserver is set properly if options.remoteWebServer == '127.0.0.1': self.error("ERROR: Either you specified the loopback for the remote webserver or ", "your local IP cannot be detected. Please provide the local ip in --remote-webserver") if not options.httpPort: options.httpPort = automation.DEFAULT_HTTP_PORT if not options.sslPort: options.sslPort = automation.DEFAULT_SSL_PORT # One of remoteAppPath (relative path to application) or the app (executable) must be # set, but not both. If both are set, we destroy the user's selection for app # so instead of silently destroying a user specificied setting, we # error. if options.remoteAppPath and options.app: self.error( "ERROR: You cannot specify both the remoteAppPath and the app") elif options.remoteAppPath: options.app = options.remoteTestRoot + "/" + options.remoteAppPath elif options.app is None: # Neither remoteAppPath nor app are set -- error self.error("ERROR: You must specify either appPath or app") if options.xrePath is None: self.error( "ERROR: You must specify the path to the controller xre directory") else: # Ensure xrepath is a full path options.xrePath = os.path.abspath(options.xrePath) options.localLogName = options.remoteLogFile options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile # Ensure that the options.logfile (which the base class uses) is set to # the remote setting when running remote. Also, if the user set the # log file name there, use that instead of reusing the remotelogfile as # above. if options.logFile: # If the user specified a local logfile name use that options.localLogName = options.logFile options.logFile = options.remoteLogFile if options.pidFile != "": with open(options.pidFile, 'w') as f: f.write(str(os.getpid())) # httpd-path is specified by standard makefile targets and may be specified # on the command line to select a particular version of httpd.js. If not # specified, try to select the one from hostutils.zip, as required in # bug 882932. if not options.httpdPath: options.httpdPath = os.path.join(options.utilityPath, "components") if not options.ignoreWindowSize: parts = automation._devicemanager.getInfo( 'screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % ( width, height)) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False return options