summaryrefslogtreecommitdiffstats
path: root/testing/marionette/harness/marionette_harness/runner/base.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/marionette/harness/marionette_harness/runner/base.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/marionette/harness/marionette_harness/runner/base.py')
-rw-r--r--testing/marionette/harness/marionette_harness/runner/base.py1076
1 files changed, 1076 insertions, 0 deletions
diff --git a/testing/marionette/harness/marionette_harness/runner/base.py b/testing/marionette/harness/marionette_harness/runner/base.py
new file mode 100644
index 000000000..72f23524b
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -0,0 +1,1076 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import random
+import re
+import socket
+import sys
+import time
+import traceback
+import unittest
+
+from argparse import ArgumentParser
+from copy import deepcopy
+
+import mozinfo
+import moznetwork
+import mozprofile
+import mozversion
+import serve
+
+from manifestparser import TestManifest
+from manifestparser.filters import tags
+from marionette_driver.marionette import Marionette
+from moztest.adapters.unit import StructuredTestResult, StructuredTestRunner
+from moztest.results import TestResult, TestResultCollection, relevant_line
+from serve import iter_proc, iter_url
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def update_mozinfo(path=None):
+ """Walk up directories to find mozinfo.json and update the info."""
+ path = path or here
+ dirs = set()
+ while path != os.path.expanduser('~'):
+ if path in dirs:
+ break
+ dirs.add(path)
+ path = os.path.split(path)[0]
+
+ return mozinfo.find_and_update_from_json(*dirs)
+
+
+class MarionetteTest(TestResult):
+
+ @property
+ def test_name(self):
+ if self.test_class is not None:
+ return '{0}.py {1}.{2}'.format(self.test_class.split('.')[0],
+ self.test_class,
+ self.name)
+ else:
+ return self.name
+
+
+class MarionetteTestResult(StructuredTestResult, TestResultCollection):
+
+ resultClass = MarionetteTest
+
+ def __init__(self, *args, **kwargs):
+ self.marionette = kwargs.pop('marionette')
+ TestResultCollection.__init__(self, 'MarionetteTest')
+ self.passed = 0
+ self.testsRun = 0
+ self.result_modifiers = [] # used by mixins to modify the result
+ StructuredTestResult.__init__(self, *args, **kwargs)
+
+ @property
+ def skipped(self):
+ return [t for t in self if t.result == 'SKIPPED']
+
+ @skipped.setter
+ def skipped(self, value):
+ pass
+
+ @property
+ def expectedFailures(self):
+ return [t for t in self if t.result == 'KNOWN-FAIL']
+
+ @expectedFailures.setter
+ def expectedFailures(self, value):
+ pass
+
+ @property
+ def unexpectedSuccesses(self):
+ return [t for t in self if t.result == 'UNEXPECTED-PASS']
+
+ @unexpectedSuccesses.setter
+ def unexpectedSuccesses(self, value):
+ pass
+
+ @property
+ def tests_passed(self):
+ return [t for t in self if t.result == 'PASS']
+
+ @property
+ def errors(self):
+ return [t for t in self if t.result == 'ERROR']
+
+ @errors.setter
+ def errors(self, value):
+ pass
+
+ @property
+ def failures(self):
+ return [t for t in self if t.result == 'UNEXPECTED-FAIL']
+
+ @failures.setter
+ def failures(self, value):
+ pass
+
+ @property
+ def duration(self):
+ if self.stop_time:
+ return self.stop_time - self.start_time
+ else:
+ return 0
+
+ def add_test_result(self, test, result_expected='PASS',
+ result_actual='PASS', output='', context=None, **kwargs):
+ def get_class(test):
+ return test.__class__.__module__ + '.' + test.__class__.__name__
+
+ name = str(test).split()[0]
+ test_class = get_class(test)
+ if hasattr(test, 'jsFile'):
+ name = os.path.basename(test.jsFile)
+ test_class = None
+
+ t = self.resultClass(name=name, test_class=test_class,
+ time_start=test.start_time, result_expected=result_expected,
+ context=context, **kwargs)
+ # call any registered result modifiers
+ for modifier in self.result_modifiers:
+ result_expected, result_actual, output, context = modifier(
+ t, result_expected, result_actual, output, context)
+ t.finish(result_actual,
+ time_end=time.time() if test.start_time else 0,
+ reason=relevant_line(output),
+ output=output)
+ self.append(t)
+
+ def addError(self, test, err):
+ self.add_test_result(test, output=self._exc_info_to_string(err, test),
+ result_actual='ERROR')
+ super(MarionetteTestResult, self).addError(test, err)
+
+ def addFailure(self, test, err):
+ self.add_test_result(test, output=self._exc_info_to_string(err, test),
+ result_actual='UNEXPECTED-FAIL')
+ super(MarionetteTestResult, self).addFailure(test, err)
+
+ def addSuccess(self, test):
+ self.passed += 1
+ self.add_test_result(test, result_actual='PASS')
+ super(MarionetteTestResult, self).addSuccess(test)
+
+ def addExpectedFailure(self, test, err):
+ """Called when an expected failure/error occured."""
+ self.add_test_result(test, output=self._exc_info_to_string(err, test),
+ result_actual='KNOWN-FAIL')
+ super(MarionetteTestResult, self).addExpectedFailure(test, err)
+
+ def addUnexpectedSuccess(self, test):
+ """Called when a test was expected to fail, but succeed."""
+ self.add_test_result(test, result_actual='UNEXPECTED-PASS')
+ super(MarionetteTestResult, self).addUnexpectedSuccess(test)
+
+ def addSkip(self, test, reason):
+ self.add_test_result(test, output=reason, result_actual='SKIPPED')
+ super(MarionetteTestResult, self).addSkip(test, reason)
+
+ def getInfo(self, test):
+ return test.test_name
+
+ def getDescription(self, test):
+ doc_first_line = test.shortDescription()
+ if self.descriptions and doc_first_line:
+ return '\n'.join((str(test), doc_first_line))
+ else:
+ desc = str(test)
+ if hasattr(test, 'jsFile'):
+ desc = "{0}, {1}".format(test.jsFile, desc)
+ return desc
+
+ def printLogs(self, test):
+ for testcase in test._tests:
+ if hasattr(testcase, 'loglines') and testcase.loglines:
+ # Don't dump loglines to the console if they only contain
+ # TEST-START and TEST-END.
+ skip_log = True
+ for line in testcase.loglines:
+ str_line = ' '.join(line)
+ if 'TEST-END' not in str_line and 'TEST-START' not in str_line:
+ skip_log = False
+ break
+ if skip_log:
+ return
+ self.logger.info('START LOG:')
+ for line in testcase.loglines:
+ self.logger.info(' '.join(line).encode('ascii', 'replace'))
+ self.logger.info('END LOG:')
+
+ def stopTest(self, *args, **kwargs):
+ unittest._TextTestResult.stopTest(self, *args, **kwargs)
+ if self.marionette.check_for_crash():
+ # this tells unittest.TestSuite not to continue running tests
+ self.shouldStop = True
+ test = next((a for a in args if isinstance(a, unittest.TestCase)),
+ None)
+ if test:
+ self.addError(test, sys.exc_info())
+
+
+class MarionetteTextTestRunner(StructuredTestRunner):
+
+ resultclass = MarionetteTestResult
+
+ def __init__(self, **kwargs):
+ self.marionette = kwargs.pop('marionette')
+ self.capabilities = kwargs.pop('capabilities')
+
+ StructuredTestRunner.__init__(self, **kwargs)
+
+ def _makeResult(self):
+ return self.resultclass(self.stream,
+ self.descriptions,
+ self.verbosity,
+ marionette=self.marionette,
+ logger=self.logger,
+ result_callbacks=self.result_callbacks)
+
+ def run(self, test):
+ result = super(MarionetteTextTestRunner, self).run(test)
+ result.printLogs(test)
+ return result
+
+
+class BaseMarionetteArguments(ArgumentParser):
+ # Bug 1336953 - Until we can remove the socket timeout parameter it has to be
+ # set a default value which is larger than the longest timeout as defined by the
+ # WebDriver spec. In that case its 300s for page load. Also add another minute
+ # so that slow builds have enough time to send the timeout error to the client.
+ socket_timeout_default = 360.0
+
+ def __init__(self, **kwargs):
+ ArgumentParser.__init__(self, **kwargs)
+
+ def dir_path(path):
+ path = os.path.abspath(os.path.expanduser(path))
+ if not os.access(path, os.F_OK):
+ os.makedirs(path)
+ return path
+
+ self.argument_containers = []
+ self.add_argument('tests',
+ nargs='*',
+ default=[],
+ help='Tests to run. '
+ 'One or more paths to test files (Python or JS), '
+ 'manifest files (.ini) or directories. '
+ 'When a directory is specified, '
+ 'all test files in the directory will be run.')
+ self.add_argument('--binary',
+ help='path to gecko executable to launch before running the test')
+ self.add_argument('--address',
+ help='host:port of running Gecko instance to connect to')
+ self.add_argument('--emulator',
+ action='store_true',
+ help='If no --address is given, then the harness will launch an '
+ 'emulator. (See Remote options group.) '
+ 'If --address is given, then the harness assumes you are '
+ 'running an emulator already, and will launch gecko app '
+ 'on that emulator.')
+ self.add_argument('--app',
+ help='application to use. see marionette_driver.geckoinstance')
+ self.add_argument('--app-arg',
+ dest='app_args',
+ action='append',
+ default=[],
+ help='specify a command line argument to be passed onto the application')
+ self.add_argument('--profile',
+ help='profile to use when launching the gecko process. If not passed, '
+ 'then a profile will be constructed and used',
+ type=dir_path)
+ self.add_argument('--pref',
+ action='append',
+ dest='prefs_args',
+ help="A preference to set. Must be a key-value pair separated by a ':'.")
+ self.add_argument('--preferences',
+ action='append',
+ dest='prefs_files',
+ help="read preferences from a JSON or INI file. For INI, use "
+ "'file.ini:section' to specify a particular section.")
+ self.add_argument('--addon',
+ action='append',
+ dest='addons',
+ help="addon to install; repeat for multiple addons.")
+ self.add_argument('--repeat',
+ type=int,
+ default=0,
+ help='number of times to repeat the test(s)')
+ self.add_argument('--testvars',
+ action='append',
+ help='path to a json file with any test data required')
+ self.add_argument('--symbols-path',
+ help='absolute path to directory containing breakpad symbols, or the '
+ 'url of a zip file containing symbols')
+ self.add_argument('--startup-timeout',
+ type=int,
+ default=60,
+ help='the max number of seconds to wait for a Marionette connection '
+ 'after launching a binary')
+ self.add_argument('--shuffle',
+ action='store_true',
+ default=False,
+ help='run tests in a random order')
+ self.add_argument('--shuffle-seed',
+ type=int,
+ default=random.randint(0, sys.maxint),
+ help='Use given seed to shuffle tests')
+ self.add_argument('--total-chunks',
+ type=int,
+ help='how many chunks to split the tests up into')
+ self.add_argument('--this-chunk',
+ type=int,
+ help='which chunk to run')
+ self.add_argument('--server-root',
+ help='url to a webserver or path to a document root from which content '
+ 'resources are served (default: {}).'.format(os.path.join(
+ os.path.dirname(here), 'www')))
+ self.add_argument('--gecko-log',
+ help="Define the path to store log file. If the path is"
+ " a directory, the real log file will be created"
+ " given the format gecko-(timestamp).log. If it is"
+ " a file, if will be used directly. '-' may be passed"
+ " to write to stdout. Default: './gecko.log'")
+ self.add_argument('--logger-name',
+ default='Marionette-based Tests',
+ help='Define the name to associate with the logger used')
+ self.add_argument('--jsdebugger',
+ action='store_true',
+ default=False,
+ help='Enable the jsdebugger for marionette javascript.')
+ self.add_argument('--pydebugger',
+ help='Enable python post-mortem debugger when a test fails.'
+ ' Pass in the debugger you want to use, eg pdb or ipdb.')
+ self.add_argument('--socket-timeout',
+ type=float,
+ default=self.socket_timeout_default,
+ help='Set the global timeout for marionette socket operations.'
+ ' Default: %(default)ss.')
+ self.add_argument('--disable-e10s',
+ action='store_false',
+ dest='e10s',
+ default=True,
+ help='Disable e10s when running marionette tests.')
+ self.add_argument('--tag',
+ action='append', dest='test_tags',
+ default=None,
+ help="Filter out tests that don't have the given tag. Can be "
+ "used multiple times in which case the test must contain "
+ "at least one of the given tags.")
+ self.add_argument('--workspace',
+ action='store',
+ default=None,
+ help="Path to directory for Marionette output. "
+ "(Default: .) (Default profile dest: TMP)",
+ type=dir_path)
+ self.add_argument('-v', '--verbose',
+ action='count',
+ help='Increase verbosity to include debug messages with -v, '
+ 'and trace messages with -vv.')
+ self.register_argument_container(RemoteMarionetteArguments())
+
+ def register_argument_container(self, container):
+ group = self.add_argument_group(container.name)
+
+ for cli, kwargs in container.args:
+ group.add_argument(*cli, **kwargs)
+
+ self.argument_containers.append(container)
+
+ def parse_known_args(self, args=None, namespace=None):
+ args, remainder = ArgumentParser.parse_known_args(self, args, namespace)
+ for container in self.argument_containers:
+ if hasattr(container, 'parse_args_handler'):
+ container.parse_args_handler(args)
+ return (args, remainder)
+
+ def _get_preferences(self, prefs_files, prefs_args):
+ """Return user defined profile preferences as a dict."""
+ # object that will hold the preferences
+ prefs = mozprofile.prefs.Preferences()
+
+ # add preferences files
+ if prefs_files:
+ for prefs_file in prefs_files:
+ prefs.add_file(prefs_file)
+
+ separator = ':'
+ cli_prefs = []
+ if prefs_args:
+ misformatted = []
+ for pref in prefs_args:
+ if separator not in pref:
+ misformatted.append(pref)
+ else:
+ cli_prefs.append(pref.split(separator, 1))
+ if misformatted:
+ self._print_message("Warning: Ignoring preferences not in key{}value format: {}\n"
+ .format(separator, ", ".join(misformatted)))
+ # string preferences
+ prefs.add(cli_prefs, cast=True)
+
+ return dict(prefs())
+
+ def verify_usage(self, args):
+ if not args.tests:
+ self.error('You must specify one or more test files, manifests, or directories.')
+
+ missing_tests = [path for path in args.tests if not os.path.exists(path)]
+ if missing_tests:
+ self.error("Test file(s) not found: " + " ".join([path for path in missing_tests]))
+
+ if not args.address and not args.binary and not args.emulator:
+ self.error('You must specify --binary, or --address, or --emulator')
+
+ if args.total_chunks is not None and args.this_chunk is None:
+ self.error('You must specify which chunk to run.')
+
+ if args.this_chunk is not None and args.total_chunks is None:
+ self.error('You must specify how many chunks to split the tests into.')
+
+ if args.total_chunks is not None:
+ if not 1 < args.total_chunks:
+ self.error('Total chunks must be greater than 1.')
+ if not 1 <= args.this_chunk <= args.total_chunks:
+ self.error('Chunk to run must be between 1 and {}.'.format(args.total_chunks))
+
+ if args.jsdebugger:
+ args.app_args.append('-jsdebugger')
+ args.socket_timeout = None
+
+ args.prefs = self._get_preferences(args.prefs_files, args.prefs_args)
+
+ for container in self.argument_containers:
+ if hasattr(container, 'verify_usage_handler'):
+ container.verify_usage_handler(args)
+
+ return args
+
+
+class RemoteMarionetteArguments(object):
+ name = 'Remote (Emulator/Device)'
+ args = [
+ [['--emulator-binary'],
+ {'help': 'Path to emulator binary. By default mozrunner uses `which emulator`',
+ 'dest': 'emulator_bin',
+ }],
+ [['--adb'],
+ {'help': 'Path to the adb. By default mozrunner uses `which adb`',
+ 'dest': 'adb_path'
+ }],
+ [['--avd'],
+ {'help': ('Name of an AVD available in your environment.'
+ 'See mozrunner.FennecEmulatorRunner'),
+ }],
+ [['--avd-home'],
+ {'help': 'Path to avd parent directory',
+ }],
+ [['--device'],
+ {'help': ('Serial ID to connect to as seen in `adb devices`,'
+ 'e.g emulator-5444'),
+ 'dest': 'device_serial',
+ }],
+ [['--package'],
+ {'help': 'Name of Android package, e.g. org.mozilla.fennec',
+ 'dest': 'package_name',
+ }],
+
+ ]
+
+
+class Fixtures(object):
+ def where_is(self, uri, on="http"):
+ return serve.where_is(uri, on)
+
+
+class BaseMarionetteTestRunner(object):
+
+ textrunnerclass = MarionetteTextTestRunner
+ driverclass = Marionette
+
+ def __init__(self, address=None,
+ app=None, app_args=None, binary=None, profile=None,
+ logger=None, logdir=None,
+ repeat=0, testvars=None,
+ symbols_path=None,
+ shuffle=False, shuffle_seed=random.randint(0, sys.maxint), this_chunk=1,
+ total_chunks=1,
+ server_root=None, gecko_log=None, result_callbacks=None,
+ prefs=None, test_tags=None,
+ socket_timeout=BaseMarionetteArguments.socket_timeout_default,
+ startup_timeout=None, addons=None, workspace=None,
+ verbose=0, e10s=True, emulator=False, **kwargs):
+ self._appinfo = None
+ self._appName = None
+ self._capabilities = None
+ self._filename_pattern = None
+ self._version_info = {}
+
+ self.fixture_servers = {}
+ self.fixtures = Fixtures()
+ self.extra_kwargs = kwargs
+ self.test_kwargs = deepcopy(kwargs)
+ self.address = address
+ self.app = app
+ self.app_args = app_args or []
+ self.bin = binary
+ self.emulator = emulator
+ self.profile = profile
+ self.addons = addons
+ self.logger = logger
+ self.marionette = None
+ self.logdir = logdir
+ self.repeat = repeat
+ self.symbols_path = symbols_path
+ self.socket_timeout = socket_timeout
+ self.shuffle = shuffle
+ self.shuffle_seed = shuffle_seed
+ self.server_root = server_root
+ self.this_chunk = this_chunk
+ self.total_chunks = total_chunks
+ self.mixin_run_tests = []
+ self.manifest_skipped_tests = []
+ self.tests = []
+ self.result_callbacks = result_callbacks or []
+ self.prefs = prefs or {}
+ self.test_tags = test_tags
+ self.startup_timeout = startup_timeout
+ self.workspace = workspace
+ # If no workspace is set, default location for gecko.log is .
+ # and default location for profile is TMP
+ self.workspace_path = workspace or os.getcwd()
+ self.verbose = verbose
+ self.e10s = e10s
+ if self.e10s:
+ self.prefs.update({
+ 'browser.tabs.remote.autostart': True,
+ 'browser.tabs.remote.force-enable': True,
+ 'extensions.e10sBlocksEnabling': False
+ })
+
+ def gather_debug(test, status):
+ # No screenshots and page source for skipped tests
+ if status == "SKIP":
+ return
+
+ rv = {}
+ marionette = test._marionette_weakref()
+
+ # In the event we're gathering debug without starting a session,
+ # skip marionette commands
+ if marionette.session is not None:
+ try:
+ with marionette.using_context(marionette.CONTEXT_CHROME):
+ rv['screenshot'] = marionette.screenshot()
+ with marionette.using_context(marionette.CONTEXT_CONTENT):
+ rv['source'] = marionette.page_source
+ except Exception as exc:
+ self.logger.warning('Failed to gather test failure debug: {}'.format(exc))
+ return rv
+
+ self.result_callbacks.append(gather_debug)
+
+ # testvars are set up in self.testvars property
+ self._testvars = None
+ self.testvars_paths = testvars
+
+ self.test_handlers = []
+
+ self.reset_test_stats()
+
+ self.logger.info('Using workspace for temporary data: '
+ '"{}"'.format(self.workspace_path))
+
+ if not gecko_log:
+ self.gecko_log = os.path.join(self.workspace_path or '', 'gecko.log')
+ else:
+ self.gecko_log = gecko_log
+
+ self.results = []
+
+ @property
+ def filename_pattern(self):
+ if self._filename_pattern is None:
+ self._filename_pattern = re.compile(
+ "^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$")
+
+ return self._filename_pattern
+
+ @property
+ def testvars(self):
+ if self._testvars is not None:
+ return self._testvars
+
+ self._testvars = {}
+
+ def update(d, u):
+ """Update a dictionary that may contain nested dictionaries."""
+ for k, v in u.iteritems():
+ o = d.get(k, {})
+ if isinstance(v, dict) and isinstance(o, dict):
+ d[k] = update(d.get(k, {}), v)
+ else:
+ d[k] = u[k]
+ return d
+
+ json_testvars = self._load_testvars()
+ for j in json_testvars:
+ self._testvars = update(self._testvars, j)
+ return self._testvars
+
+ def _load_testvars(self):
+ data = []
+ if self.testvars_paths is not None:
+ for path in list(self.testvars_paths):
+ path = os.path.abspath(os.path.expanduser(path))
+ if not os.path.exists(path):
+ raise IOError('--testvars file {} does not exist'.format(path))
+ try:
+ with open(path) as f:
+ data.append(json.loads(f.read()))
+ except ValueError as e:
+ exc, val, tb = sys.exc_info()
+ msg = "JSON file ({0}) is not properly formatted: {1}"
+ raise exc, msg.format(os.path.abspath(path), e.message), tb
+ return data
+
+ @property
+ def capabilities(self):
+ if self._capabilities:
+ return self._capabilities
+
+ self.marionette.start_session()
+ self._capabilities = self.marionette.session_capabilities
+ self.marionette.delete_session()
+ return self._capabilities
+
+ @property
+ def appinfo(self):
+ if self._appinfo:
+ return self._appinfo
+
+ self.marionette.start_session()
+ with self.marionette.using_context('chrome'):
+ self._appinfo = self.marionette.execute_script("""
+ try {
+ return Services.appinfo;
+ } catch (e) {
+ return null;
+ }""")
+ self.marionette.delete_session()
+ self._appinfo = self._appinfo or {}
+ return self._appinfo
+
+ @property
+ def appName(self):
+ if self._appName:
+ return self._appName
+
+ self._appName = self.capabilities.get('browserName')
+ return self._appName
+
+ @property
+ def bin(self):
+ return self._bin
+
+ @bin.setter
+ def bin(self, path):
+ """Set binary and reset parts of runner accordingly.
+ Intended use: to change binary between calls to run_tests
+ """
+ self._bin = path
+ self.tests = []
+ self.cleanup()
+
+ @property
+ def version_info(self):
+ if not self._version_info:
+ try:
+ # TODO: Get version_info in Fennec case
+ self._version_info = mozversion.get_version(binary=self.bin)
+ except Exception:
+ self.logger.warning("Failed to retrieve version information for {}".format(
+ self.bin))
+ return self._version_info
+
+ def reset_test_stats(self):
+ self.passed = 0
+ self.failed = 0
+ self.crashed = 0
+ self.unexpected_successes = 0
+ self.todo = 0
+ self.skipped = 0
+ self.failures = []
+
+ def _build_kwargs(self):
+ if self.logdir and not os.access(self.logdir, os.F_OK):
+ os.mkdir(self.logdir)
+
+ kwargs = {
+ 'socket_timeout': self.socket_timeout,
+ 'prefs': self.prefs,
+ 'startup_timeout': self.startup_timeout,
+ 'verbose': self.verbose,
+ 'symbols_path': self.symbols_path,
+ }
+ if self.bin or self.emulator:
+ kwargs.update({
+ 'host': 'localhost',
+ 'port': 2828,
+ 'app': self.app,
+ 'app_args': self.app_args,
+ 'profile': self.profile,
+ 'addons': self.addons,
+ 'gecko_log': self.gecko_log,
+ # ensure Marionette class takes care of starting gecko instance
+ 'bin': True,
+ })
+
+ if self.bin:
+ kwargs.update({
+ 'bin': self.bin,
+ })
+
+ if self.emulator:
+ kwargs.update({
+ 'avd_home': self.extra_kwargs.get('avd_home'),
+ 'adb_path': self.extra_kwargs.get('adb_path'),
+ 'emulator_binary': self.extra_kwargs.get('emulator_bin'),
+ 'avd': self.extra_kwargs.get('avd'),
+ 'package_name': self.extra_kwargs.get('package_name'),
+ })
+
+ if self.address:
+ host, port = self.address.split(':')
+ kwargs.update({
+ 'host': host,
+ 'port': int(port),
+ })
+ if self.emulator:
+ kwargs.update({
+ 'connect_to_running_emulator': True,
+ })
+ if not self.bin and not self.emulator:
+ try:
+ # Establish a socket connection so we can vertify the data come back
+ connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ connection.connect((host, int(port)))
+ connection.close()
+ except Exception as e:
+ exc, val, tb = sys.exc_info()
+ msg = "Connection attempt to {0}:{1} failed with error: {2}"
+ raise exc, msg.format(host, port, e), tb
+ if self.workspace:
+ kwargs['workspace'] = self.workspace_path
+ return kwargs
+
+ def record_crash(self):
+ crash = True
+ try:
+ crash = self.marionette.check_for_crash()
+ self.crashed += int(crash)
+ except Exception:
+ traceback.print_exc()
+ return crash
+
+ def _initialize_test_run(self, tests):
+ assert len(tests) > 0
+ assert len(self.test_handlers) > 0
+ self.reset_test_stats()
+
+ def _add_tests(self, tests):
+ for test in tests:
+ self.add_test(test)
+
+ invalid_tests = [t['filepath'] for t in self.tests
+ if not self._is_filename_valid(t['filepath'])]
+ if invalid_tests:
+ raise Exception("Test file names must be of the form "
+ "'test_something.py', 'test_something.js', or 'testSomething.js'."
+ " Invalid test names:\n {}".format('\n '.join(invalid_tests)))
+
+ def _is_filename_valid(self, filename):
+ filename = os.path.basename(filename)
+ return self.filename_pattern.match(filename)
+
+ def _log_skipped_tests(self):
+ for test in self.manifest_skipped_tests:
+ name = os.path.basename(test['path'])
+ self.logger.test_start(name)
+ self.logger.test_end(name,
+ 'SKIP',
+ message=test['disabled'])
+ self.todo += 1
+
+ def run_tests(self, tests):
+ start_time = time.time()
+ self._initialize_test_run(tests)
+
+ if self.marionette is None:
+ self.marionette = self.driverclass(**self._build_kwargs())
+ self.logger.info("Profile path is %s" % self.marionette.profile_path)
+
+ if len(self.fixture_servers) == 0 or \
+ any(not server.is_alive for _, server in self.fixture_servers):
+ self.logger.info("Starting fixture servers")
+ self.fixture_servers = self.start_fixture_servers()
+ for url in iter_url(self.fixture_servers):
+ self.logger.info("Fixture server listening on %s" % url)
+
+ # backwards compatibility
+ self.marionette.baseurl = serve.where_is("/")
+
+ self._add_tests(tests)
+
+ device_info = None
+ if self.marionette.instance and self.emulator:
+ try:
+ device_info = self.marionette.instance.runner.device.dm.getInfo()
+ except Exception:
+ self.logger.warning('Could not get device info', exc_info=True)
+
+ appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False)
+ self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled"))
+ if self.e10s != appinfo_e10s:
+ message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does "
+ "not match browser appinfo")
+ self.cleanup()
+ raise AssertionError(message_e10s)
+
+ self.logger.suite_start(self.tests,
+ version_info=self.version_info,
+ device_info=device_info)
+
+ self._log_skipped_tests()
+
+ interrupted = None
+ try:
+ counter = self.repeat
+ while counter >= 0:
+ round_num = self.repeat - counter
+ if round_num > 0:
+ self.logger.info('\nREPEAT {}\n-------'.format(round_num))
+ self.run_test_sets()
+ counter -= 1
+ except KeyboardInterrupt:
+ # in case of KeyboardInterrupt during the test execution
+ # we want to display current test results.
+ # so we keep the exception to raise it later.
+ interrupted = sys.exc_info()
+ except:
+ # For any other exception we return immediately and have to
+ # cleanup running processes
+ self.cleanup()
+ raise
+
+ try:
+ self._print_summary(tests)
+ self.record_crash()
+ self.elapsedtime = time.time() - start_time
+
+ for run_tests in self.mixin_run_tests:
+ run_tests(tests)
+ if self.shuffle:
+ self.logger.info("Using shuffle seed: %d" % self.shuffle_seed)
+
+ self.logger.suite_end()
+ except:
+ # raise only the exception if we were not interrupted
+ if not interrupted:
+ raise
+ finally:
+ self.cleanup()
+
+ # reraise previous interruption now
+ if interrupted:
+ raise interrupted[0], interrupted[1], interrupted[2]
+
+ def _print_summary(self, tests):
+ self.logger.info('\nSUMMARY\n-------')
+ self.logger.info('passed: {}'.format(self.passed))
+ if self.unexpected_successes == 0:
+ self.logger.info('failed: {}'.format(self.failed))
+ else:
+ self.logger.info(
+ 'failed: {0} (unexpected sucesses: {1})'.format(self.failed,
+ self.unexpected_successes))
+ if self.skipped == 0:
+ self.logger.info('todo: {}'.format(self.todo))
+ else:
+ self.logger.info('todo: {0} (skipped: {1})'.format(self.todo, self.skipped))
+
+ if self.failed > 0:
+ self.logger.info('\nFAILED TESTS\n-------')
+ for failed_test in self.failures:
+ self.logger.info('{}'.format(failed_test[0]))
+
+ def start_fixture_servers(self):
+ root = self.server_root or os.path.join(os.path.dirname(here), "www")
+ if self.appName == "fennec":
+ return serve.start(root, host=moznetwork.get_ip())
+ else:
+ return serve.start(root)
+
+ def add_test(self, test, expected='pass'):
+ filepath = os.path.abspath(test)
+
+ if os.path.isdir(filepath):
+ for root, dirs, files in os.walk(filepath):
+ for filename in files:
+ if filename.endswith('.ini'):
+ msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
+ " See --help for details.")
+ relpath = os.path.relpath(os.path.join(root, filename), filepath)
+ self.logger.warning(msg_tmpl.format(relpath, filepath))
+ elif self._is_filename_valid(filename):
+ test_file = os.path.join(root, filename)
+ self.add_test(test_file)
+ return
+
+ file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
+
+ if file_ext == '.ini':
+ manifest = TestManifest()
+ manifest.read(filepath)
+
+ json_path = update_mozinfo(filepath)
+ self.logger.info("mozinfo updated from: {}".format(json_path))
+ self.logger.info("mozinfo is: {}".format(mozinfo.info))
+
+ filters = []
+ if self.test_tags:
+ filters.append(tags(self.test_tags))
+
+ values = {
+ "appname": self.appName,
+ "e10s": self.e10s,
+ "manage_instance": self.marionette.instance is not None,
+ }
+ values.update(mozinfo.info)
+
+ manifest_tests = manifest.active_tests(exists=False,
+ disabled=True,
+ filters=filters,
+ **values)
+ if len(manifest_tests) == 0:
+ self.logger.error("No tests to run using specified "
+ "combination of filters: {}".format(
+ manifest.fmt_filters()))
+
+ target_tests = []
+ for test in manifest_tests:
+ if test.get('disabled'):
+ self.manifest_skipped_tests.append(test)
+ else:
+ target_tests.append(test)
+
+ for i in target_tests:
+ if not os.path.exists(i["path"]):
+ raise IOError("test file: {} does not exist".format(i["path"]))
+
+ file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
+
+ self.add_test(i["path"], i["expected"])
+ return
+
+ self.tests.append({'filepath': filepath, 'expected': expected})
+
+ def run_test(self, filepath, expected):
+ testloader = unittest.TestLoader()
+ suite = unittest.TestSuite()
+ self.test_kwargs['expected'] = expected
+ mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
+ for handler in self.test_handlers:
+ if handler.match(os.path.basename(filepath)):
+ handler.add_tests_to_suite(mod_name,
+ filepath,
+ suite,
+ testloader,
+ self.marionette,
+ self.fixtures,
+ self.testvars,
+ **self.test_kwargs)
+ break
+
+ if suite.countTestCases():
+ runner = self.textrunnerclass(logger=self.logger,
+ marionette=self.marionette,
+ capabilities=self.capabilities,
+ result_callbacks=self.result_callbacks)
+
+ results = runner.run(suite)
+ self.results.append(results)
+
+ self.failed += len(results.failures) + len(results.errors)
+ if hasattr(results, 'skipped'):
+ self.skipped += len(results.skipped)
+ self.todo += len(results.skipped)
+ self.passed += results.passed
+ for failure in results.failures + results.errors:
+ self.failures.append((results.getInfo(failure), failure.output,
+ 'TEST-UNEXPECTED-FAIL'))
+ if hasattr(results, 'unexpectedSuccesses'):
+ self.failed += len(results.unexpectedSuccesses)
+ self.unexpected_successes += len(results.unexpectedSuccesses)
+ for failure in results.unexpectedSuccesses:
+ self.failures.append((results.getInfo(failure), failure.output,
+ 'TEST-UNEXPECTED-PASS'))
+ if hasattr(results, 'expectedFailures'):
+ self.todo += len(results.expectedFailures)
+
+ self.mixin_run_tests = []
+ for result in self.results:
+ result.result_modifiers = []
+
+ def run_test_set(self, tests):
+ if self.shuffle:
+ random.seed(self.shuffle_seed)
+ random.shuffle(tests)
+
+ for test in tests:
+ self.run_test(test['filepath'], test['expected'])
+ if self.record_crash():
+ break
+
+ def run_test_sets(self):
+ if len(self.tests) < 1:
+ raise Exception('There are no tests to run.')
+ elif self.total_chunks > len(self.tests):
+ raise ValueError('Total number of chunks must be between 1 and {}.'
+ .format(len(self.tests)))
+ if self.total_chunks > 1:
+ chunks = [[] for i in range(self.total_chunks)]
+ for i, test in enumerate(self.tests):
+ target_chunk = i % self.total_chunks
+ chunks[target_chunk].append(test)
+
+ self.logger.info('Running chunk {0} of {1} ({2} tests selected from a '
+ 'total of {3})'.format(self.this_chunk, self.total_chunks,
+ len(chunks[self.this_chunk - 1]),
+ len(self.tests)))
+ self.tests = chunks[self.this_chunk - 1]
+
+ self.run_test_set(self.tests)
+
+ def cleanup(self):
+ for proc in iter_proc(self.fixture_servers):
+ proc.stop()
+ proc.kill()
+ self.fixture_servers = {}
+
+ if hasattr(self, 'marionette') and self.marionette:
+ if self.marionette.instance is not None:
+ self.marionette.instance.close()
+ self.marionette.instance = None
+
+ self.marionette.cleanup()
+ self.marionette = None
+
+ __del__ = cleanup