diff options
Diffstat (limited to 'testing/marionette/harness/marionette_harness/runner/base.py')
| -rw-r--r-- | testing/marionette/harness/marionette_harness/runner/base.py | 1076 |
1 files changed, 1076 insertions, 0 deletions
diff --git a/testing/marionette/harness/marionette_harness/runner/base.py b/testing/marionette/harness/marionette_harness/runner/base.py new file mode 100644 index 000000000..72f23524b --- /dev/null +++ b/testing/marionette/harness/marionette_harness/runner/base.py @@ -0,0 +1,1076 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import random +import re +import socket +import sys +import time +import traceback +import unittest + +from argparse import ArgumentParser +from copy import deepcopy + +import mozinfo +import moznetwork +import mozprofile +import mozversion +import serve + +from manifestparser import TestManifest +from manifestparser.filters import tags +from marionette_driver.marionette import Marionette +from moztest.adapters.unit import StructuredTestResult, StructuredTestRunner +from moztest.results import TestResult, TestResultCollection, relevant_line +from serve import iter_proc, iter_url + +here = os.path.abspath(os.path.dirname(__file__)) + + +def update_mozinfo(path=None): + """Walk up directories to find mozinfo.json and update the info.""" + path = path or here + dirs = set() + while path != os.path.expanduser('~'): + if path in dirs: + break + dirs.add(path) + path = os.path.split(path)[0] + + return mozinfo.find_and_update_from_json(*dirs) + + +class MarionetteTest(TestResult): + + @property + def test_name(self): + if self.test_class is not None: + return '{0}.py {1}.{2}'.format(self.test_class.split('.')[0], + self.test_class, + self.name) + else: + return self.name + + +class MarionetteTestResult(StructuredTestResult, TestResultCollection): + + resultClass = MarionetteTest + + def __init__(self, *args, **kwargs): + self.marionette = kwargs.pop('marionette') + TestResultCollection.__init__(self, 'MarionetteTest') + self.passed = 0 + self.testsRun = 0 + self.result_modifiers = [] # used by mixins to modify the result + StructuredTestResult.__init__(self, *args, **kwargs) + + @property + def skipped(self): + return [t for t in self if t.result == 'SKIPPED'] + + @skipped.setter + def skipped(self, value): + pass + + @property + def expectedFailures(self): + return [t for t in self if t.result == 'KNOWN-FAIL'] + + @expectedFailures.setter + def expectedFailures(self, value): + pass + + @property + def unexpectedSuccesses(self): + return [t for t in self if t.result == 'UNEXPECTED-PASS'] + + @unexpectedSuccesses.setter + def unexpectedSuccesses(self, value): + pass + + @property + def tests_passed(self): + return [t for t in self if t.result == 'PASS'] + + @property + def errors(self): + return [t for t in self if t.result == 'ERROR'] + + @errors.setter + def errors(self, value): + pass + + @property + def failures(self): + return [t for t in self if t.result == 'UNEXPECTED-FAIL'] + + @failures.setter + def failures(self, value): + pass + + @property + def duration(self): + if self.stop_time: + return self.stop_time - self.start_time + else: + return 0 + + def add_test_result(self, test, result_expected='PASS', + result_actual='PASS', output='', context=None, **kwargs): + def get_class(test): + return test.__class__.__module__ + '.' + test.__class__.__name__ + + name = str(test).split()[0] + test_class = get_class(test) + if hasattr(test, 'jsFile'): + name = os.path.basename(test.jsFile) + test_class = None + + t = self.resultClass(name=name, test_class=test_class, + time_start=test.start_time, result_expected=result_expected, + context=context, **kwargs) + # call any registered result modifiers + for modifier in self.result_modifiers: + result_expected, result_actual, output, context = modifier( + t, result_expected, result_actual, output, context) + t.finish(result_actual, + time_end=time.time() if test.start_time else 0, + reason=relevant_line(output), + output=output) + self.append(t) + + def addError(self, test, err): + self.add_test_result(test, output=self._exc_info_to_string(err, test), + result_actual='ERROR') + super(MarionetteTestResult, self).addError(test, err) + + def addFailure(self, test, err): + self.add_test_result(test, output=self._exc_info_to_string(err, test), + result_actual='UNEXPECTED-FAIL') + super(MarionetteTestResult, self).addFailure(test, err) + + def addSuccess(self, test): + self.passed += 1 + self.add_test_result(test, result_actual='PASS') + super(MarionetteTestResult, self).addSuccess(test) + + def addExpectedFailure(self, test, err): + """Called when an expected failure/error occured.""" + self.add_test_result(test, output=self._exc_info_to_string(err, test), + result_actual='KNOWN-FAIL') + super(MarionetteTestResult, self).addExpectedFailure(test, err) + + def addUnexpectedSuccess(self, test): + """Called when a test was expected to fail, but succeed.""" + self.add_test_result(test, result_actual='UNEXPECTED-PASS') + super(MarionetteTestResult, self).addUnexpectedSuccess(test) + + def addSkip(self, test, reason): + self.add_test_result(test, output=reason, result_actual='SKIPPED') + super(MarionetteTestResult, self).addSkip(test, reason) + + def getInfo(self, test): + return test.test_name + + def getDescription(self, test): + doc_first_line = test.shortDescription() + if self.descriptions and doc_first_line: + return '\n'.join((str(test), doc_first_line)) + else: + desc = str(test) + if hasattr(test, 'jsFile'): + desc = "{0}, {1}".format(test.jsFile, desc) + return desc + + def printLogs(self, test): + for testcase in test._tests: + if hasattr(testcase, 'loglines') and testcase.loglines: + # Don't dump loglines to the console if they only contain + # TEST-START and TEST-END. + skip_log = True + for line in testcase.loglines: + str_line = ' '.join(line) + if 'TEST-END' not in str_line and 'TEST-START' not in str_line: + skip_log = False + break + if skip_log: + return + self.logger.info('START LOG:') + for line in testcase.loglines: + self.logger.info(' '.join(line).encode('ascii', 'replace')) + self.logger.info('END LOG:') + + def stopTest(self, *args, **kwargs): + unittest._TextTestResult.stopTest(self, *args, **kwargs) + if self.marionette.check_for_crash(): + # this tells unittest.TestSuite not to continue running tests + self.shouldStop = True + test = next((a for a in args if isinstance(a, unittest.TestCase)), + None) + if test: + self.addError(test, sys.exc_info()) + + +class MarionetteTextTestRunner(StructuredTestRunner): + + resultclass = MarionetteTestResult + + def __init__(self, **kwargs): + self.marionette = kwargs.pop('marionette') + self.capabilities = kwargs.pop('capabilities') + + StructuredTestRunner.__init__(self, **kwargs) + + def _makeResult(self): + return self.resultclass(self.stream, + self.descriptions, + self.verbosity, + marionette=self.marionette, + logger=self.logger, + result_callbacks=self.result_callbacks) + + def run(self, test): + result = super(MarionetteTextTestRunner, self).run(test) + result.printLogs(test) + return result + + +class BaseMarionetteArguments(ArgumentParser): + # Bug 1336953 - Until we can remove the socket timeout parameter it has to be + # set a default value which is larger than the longest timeout as defined by the + # WebDriver spec. In that case its 300s for page load. Also add another minute + # so that slow builds have enough time to send the timeout error to the client. + socket_timeout_default = 360.0 + + def __init__(self, **kwargs): + ArgumentParser.__init__(self, **kwargs) + + def dir_path(path): + path = os.path.abspath(os.path.expanduser(path)) + if not os.access(path, os.F_OK): + os.makedirs(path) + return path + + self.argument_containers = [] + self.add_argument('tests', + nargs='*', + default=[], + help='Tests to run. ' + 'One or more paths to test files (Python or JS), ' + 'manifest files (.ini) or directories. ' + 'When a directory is specified, ' + 'all test files in the directory will be run.') + self.add_argument('--binary', + help='path to gecko executable to launch before running the test') + self.add_argument('--address', + help='host:port of running Gecko instance to connect to') + self.add_argument('--emulator', + action='store_true', + help='If no --address is given, then the harness will launch an ' + 'emulator. (See Remote options group.) ' + 'If --address is given, then the harness assumes you are ' + 'running an emulator already, and will launch gecko app ' + 'on that emulator.') + self.add_argument('--app', + help='application to use. see marionette_driver.geckoinstance') + self.add_argument('--app-arg', + dest='app_args', + action='append', + default=[], + help='specify a command line argument to be passed onto the application') + self.add_argument('--profile', + help='profile to use when launching the gecko process. If not passed, ' + 'then a profile will be constructed and used', + type=dir_path) + self.add_argument('--pref', + action='append', + dest='prefs_args', + help="A preference to set. Must be a key-value pair separated by a ':'.") + self.add_argument('--preferences', + action='append', + dest='prefs_files', + help="read preferences from a JSON or INI file. For INI, use " + "'file.ini:section' to specify a particular section.") + self.add_argument('--addon', + action='append', + dest='addons', + help="addon to install; repeat for multiple addons.") + self.add_argument('--repeat', + type=int, + default=0, + help='number of times to repeat the test(s)') + self.add_argument('--testvars', + action='append', + help='path to a json file with any test data required') + self.add_argument('--symbols-path', + help='absolute path to directory containing breakpad symbols, or the ' + 'url of a zip file containing symbols') + self.add_argument('--startup-timeout', + type=int, + default=60, + help='the max number of seconds to wait for a Marionette connection ' + 'after launching a binary') + self.add_argument('--shuffle', + action='store_true', + default=False, + help='run tests in a random order') + self.add_argument('--shuffle-seed', + type=int, + default=random.randint(0, sys.maxint), + help='Use given seed to shuffle tests') + self.add_argument('--total-chunks', + type=int, + help='how many chunks to split the tests up into') + self.add_argument('--this-chunk', + type=int, + help='which chunk to run') + self.add_argument('--server-root', + help='url to a webserver or path to a document root from which content ' + 'resources are served (default: {}).'.format(os.path.join( + os.path.dirname(here), 'www'))) + self.add_argument('--gecko-log', + help="Define the path to store log file. If the path is" + " a directory, the real log file will be created" + " given the format gecko-(timestamp).log. If it is" + " a file, if will be used directly. '-' may be passed" + " to write to stdout. Default: './gecko.log'") + self.add_argument('--logger-name', + default='Marionette-based Tests', + help='Define the name to associate with the logger used') + self.add_argument('--jsdebugger', + action='store_true', + default=False, + help='Enable the jsdebugger for marionette javascript.') + self.add_argument('--pydebugger', + help='Enable python post-mortem debugger when a test fails.' + ' Pass in the debugger you want to use, eg pdb or ipdb.') + self.add_argument('--socket-timeout', + type=float, + default=self.socket_timeout_default, + help='Set the global timeout for marionette socket operations.' + ' Default: %(default)ss.') + self.add_argument('--disable-e10s', + action='store_false', + dest='e10s', + default=True, + help='Disable e10s when running marionette tests.') + self.add_argument('--tag', + action='append', dest='test_tags', + default=None, + help="Filter out tests that don't have the given tag. Can be " + "used multiple times in which case the test must contain " + "at least one of the given tags.") + self.add_argument('--workspace', + action='store', + default=None, + help="Path to directory for Marionette output. " + "(Default: .) (Default profile dest: TMP)", + type=dir_path) + self.add_argument('-v', '--verbose', + action='count', + help='Increase verbosity to include debug messages with -v, ' + 'and trace messages with -vv.') + self.register_argument_container(RemoteMarionetteArguments()) + + def register_argument_container(self, container): + group = self.add_argument_group(container.name) + + for cli, kwargs in container.args: + group.add_argument(*cli, **kwargs) + + self.argument_containers.append(container) + + def parse_known_args(self, args=None, namespace=None): + args, remainder = ArgumentParser.parse_known_args(self, args, namespace) + for container in self.argument_containers: + if hasattr(container, 'parse_args_handler'): + container.parse_args_handler(args) + return (args, remainder) + + def _get_preferences(self, prefs_files, prefs_args): + """Return user defined profile preferences as a dict.""" + # object that will hold the preferences + prefs = mozprofile.prefs.Preferences() + + # add preferences files + if prefs_files: + for prefs_file in prefs_files: + prefs.add_file(prefs_file) + + separator = ':' + cli_prefs = [] + if prefs_args: + misformatted = [] + for pref in prefs_args: + if separator not in pref: + misformatted.append(pref) + else: + cli_prefs.append(pref.split(separator, 1)) + if misformatted: + self._print_message("Warning: Ignoring preferences not in key{}value format: {}\n" + .format(separator, ", ".join(misformatted))) + # string preferences + prefs.add(cli_prefs, cast=True) + + return dict(prefs()) + + def verify_usage(self, args): + if not args.tests: + self.error('You must specify one or more test files, manifests, or directories.') + + missing_tests = [path for path in args.tests if not os.path.exists(path)] + if missing_tests: + self.error("Test file(s) not found: " + " ".join([path for path in missing_tests])) + + if not args.address and not args.binary and not args.emulator: + self.error('You must specify --binary, or --address, or --emulator') + + if args.total_chunks is not None and args.this_chunk is None: + self.error('You must specify which chunk to run.') + + if args.this_chunk is not None and args.total_chunks is None: + self.error('You must specify how many chunks to split the tests into.') + + if args.total_chunks is not None: + if not 1 < args.total_chunks: + self.error('Total chunks must be greater than 1.') + if not 1 <= args.this_chunk <= args.total_chunks: + self.error('Chunk to run must be between 1 and {}.'.format(args.total_chunks)) + + if args.jsdebugger: + args.app_args.append('-jsdebugger') + args.socket_timeout = None + + args.prefs = self._get_preferences(args.prefs_files, args.prefs_args) + + for container in self.argument_containers: + if hasattr(container, 'verify_usage_handler'): + container.verify_usage_handler(args) + + return args + + +class RemoteMarionetteArguments(object): + name = 'Remote (Emulator/Device)' + args = [ + [['--emulator-binary'], + {'help': 'Path to emulator binary. By default mozrunner uses `which emulator`', + 'dest': 'emulator_bin', + }], + [['--adb'], + {'help': 'Path to the adb. By default mozrunner uses `which adb`', + 'dest': 'adb_path' + }], + [['--avd'], + {'help': ('Name of an AVD available in your environment.' + 'See mozrunner.FennecEmulatorRunner'), + }], + [['--avd-home'], + {'help': 'Path to avd parent directory', + }], + [['--device'], + {'help': ('Serial ID to connect to as seen in `adb devices`,' + 'e.g emulator-5444'), + 'dest': 'device_serial', + }], + [['--package'], + {'help': 'Name of Android package, e.g. org.mozilla.fennec', + 'dest': 'package_name', + }], + + ] + + +class Fixtures(object): + def where_is(self, uri, on="http"): + return serve.where_is(uri, on) + + +class BaseMarionetteTestRunner(object): + + textrunnerclass = MarionetteTextTestRunner + driverclass = Marionette + + def __init__(self, address=None, + app=None, app_args=None, binary=None, profile=None, + logger=None, logdir=None, + repeat=0, testvars=None, + symbols_path=None, + shuffle=False, shuffle_seed=random.randint(0, sys.maxint), this_chunk=1, + total_chunks=1, + server_root=None, gecko_log=None, result_callbacks=None, + prefs=None, test_tags=None, + socket_timeout=BaseMarionetteArguments.socket_timeout_default, + startup_timeout=None, addons=None, workspace=None, + verbose=0, e10s=True, emulator=False, **kwargs): + self._appinfo = None + self._appName = None + self._capabilities = None + self._filename_pattern = None + self._version_info = {} + + self.fixture_servers = {} + self.fixtures = Fixtures() + self.extra_kwargs = kwargs + self.test_kwargs = deepcopy(kwargs) + self.address = address + self.app = app + self.app_args = app_args or [] + self.bin = binary + self.emulator = emulator + self.profile = profile + self.addons = addons + self.logger = logger + self.marionette = None + self.logdir = logdir + self.repeat = repeat + self.symbols_path = symbols_path + self.socket_timeout = socket_timeout + self.shuffle = shuffle + self.shuffle_seed = shuffle_seed + self.server_root = server_root + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.mixin_run_tests = [] + self.manifest_skipped_tests = [] + self.tests = [] + self.result_callbacks = result_callbacks or [] + self.prefs = prefs or {} + self.test_tags = test_tags + self.startup_timeout = startup_timeout + self.workspace = workspace + # If no workspace is set, default location for gecko.log is . + # and default location for profile is TMP + self.workspace_path = workspace or os.getcwd() + self.verbose = verbose + self.e10s = e10s + if self.e10s: + self.prefs.update({ + 'browser.tabs.remote.autostart': True, + 'browser.tabs.remote.force-enable': True, + 'extensions.e10sBlocksEnabling': False + }) + + def gather_debug(test, status): + # No screenshots and page source for skipped tests + if status == "SKIP": + return + + rv = {} + marionette = test._marionette_weakref() + + # In the event we're gathering debug without starting a session, + # skip marionette commands + if marionette.session is not None: + try: + with marionette.using_context(marionette.CONTEXT_CHROME): + rv['screenshot'] = marionette.screenshot() + with marionette.using_context(marionette.CONTEXT_CONTENT): + rv['source'] = marionette.page_source + except Exception as exc: + self.logger.warning('Failed to gather test failure debug: {}'.format(exc)) + return rv + + self.result_callbacks.append(gather_debug) + + # testvars are set up in self.testvars property + self._testvars = None + self.testvars_paths = testvars + + self.test_handlers = [] + + self.reset_test_stats() + + self.logger.info('Using workspace for temporary data: ' + '"{}"'.format(self.workspace_path)) + + if not gecko_log: + self.gecko_log = os.path.join(self.workspace_path or '', 'gecko.log') + else: + self.gecko_log = gecko_log + + self.results = [] + + @property + def filename_pattern(self): + if self._filename_pattern is None: + self._filename_pattern = re.compile( + "^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$") + + return self._filename_pattern + + @property + def testvars(self): + if self._testvars is not None: + return self._testvars + + self._testvars = {} + + def update(d, u): + """Update a dictionary that may contain nested dictionaries.""" + for k, v in u.iteritems(): + o = d.get(k, {}) + if isinstance(v, dict) and isinstance(o, dict): + d[k] = update(d.get(k, {}), v) + else: + d[k] = u[k] + return d + + json_testvars = self._load_testvars() + for j in json_testvars: + self._testvars = update(self._testvars, j) + return self._testvars + + def _load_testvars(self): + data = [] + if self.testvars_paths is not None: + for path in list(self.testvars_paths): + path = os.path.abspath(os.path.expanduser(path)) + if not os.path.exists(path): + raise IOError('--testvars file {} does not exist'.format(path)) + try: + with open(path) as f: + data.append(json.loads(f.read())) + except ValueError as e: + exc, val, tb = sys.exc_info() + msg = "JSON file ({0}) is not properly formatted: {1}" + raise exc, msg.format(os.path.abspath(path), e.message), tb + return data + + @property + def capabilities(self): + if self._capabilities: + return self._capabilities + + self.marionette.start_session() + self._capabilities = self.marionette.session_capabilities + self.marionette.delete_session() + return self._capabilities + + @property + def appinfo(self): + if self._appinfo: + return self._appinfo + + self.marionette.start_session() + with self.marionette.using_context('chrome'): + self._appinfo = self.marionette.execute_script(""" + try { + return Services.appinfo; + } catch (e) { + return null; + }""") + self.marionette.delete_session() + self._appinfo = self._appinfo or {} + return self._appinfo + + @property + def appName(self): + if self._appName: + return self._appName + + self._appName = self.capabilities.get('browserName') + return self._appName + + @property + def bin(self): + return self._bin + + @bin.setter + def bin(self, path): + """Set binary and reset parts of runner accordingly. + Intended use: to change binary between calls to run_tests + """ + self._bin = path + self.tests = [] + self.cleanup() + + @property + def version_info(self): + if not self._version_info: + try: + # TODO: Get version_info in Fennec case + self._version_info = mozversion.get_version(binary=self.bin) + except Exception: + self.logger.warning("Failed to retrieve version information for {}".format( + self.bin)) + return self._version_info + + def reset_test_stats(self): + self.passed = 0 + self.failed = 0 + self.crashed = 0 + self.unexpected_successes = 0 + self.todo = 0 + self.skipped = 0 + self.failures = [] + + def _build_kwargs(self): + if self.logdir and not os.access(self.logdir, os.F_OK): + os.mkdir(self.logdir) + + kwargs = { + 'socket_timeout': self.socket_timeout, + 'prefs': self.prefs, + 'startup_timeout': self.startup_timeout, + 'verbose': self.verbose, + 'symbols_path': self.symbols_path, + } + if self.bin or self.emulator: + kwargs.update({ + 'host': 'localhost', + 'port': 2828, + 'app': self.app, + 'app_args': self.app_args, + 'profile': self.profile, + 'addons': self.addons, + 'gecko_log': self.gecko_log, + # ensure Marionette class takes care of starting gecko instance + 'bin': True, + }) + + if self.bin: + kwargs.update({ + 'bin': self.bin, + }) + + if self.emulator: + kwargs.update({ + 'avd_home': self.extra_kwargs.get('avd_home'), + 'adb_path': self.extra_kwargs.get('adb_path'), + 'emulator_binary': self.extra_kwargs.get('emulator_bin'), + 'avd': self.extra_kwargs.get('avd'), + 'package_name': self.extra_kwargs.get('package_name'), + }) + + if self.address: + host, port = self.address.split(':') + kwargs.update({ + 'host': host, + 'port': int(port), + }) + if self.emulator: + kwargs.update({ + 'connect_to_running_emulator': True, + }) + if not self.bin and not self.emulator: + try: + # Establish a socket connection so we can vertify the data come back + connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connection.connect((host, int(port))) + connection.close() + except Exception as e: + exc, val, tb = sys.exc_info() + msg = "Connection attempt to {0}:{1} failed with error: {2}" + raise exc, msg.format(host, port, e), tb + if self.workspace: + kwargs['workspace'] = self.workspace_path + return kwargs + + def record_crash(self): + crash = True + try: + crash = self.marionette.check_for_crash() + self.crashed += int(crash) + except Exception: + traceback.print_exc() + return crash + + def _initialize_test_run(self, tests): + assert len(tests) > 0 + assert len(self.test_handlers) > 0 + self.reset_test_stats() + + def _add_tests(self, tests): + for test in tests: + self.add_test(test) + + invalid_tests = [t['filepath'] for t in self.tests + if not self._is_filename_valid(t['filepath'])] + if invalid_tests: + raise Exception("Test file names must be of the form " + "'test_something.py', 'test_something.js', or 'testSomething.js'." + " Invalid test names:\n {}".format('\n '.join(invalid_tests))) + + def _is_filename_valid(self, filename): + filename = os.path.basename(filename) + return self.filename_pattern.match(filename) + + def _log_skipped_tests(self): + for test in self.manifest_skipped_tests: + name = os.path.basename(test['path']) + self.logger.test_start(name) + self.logger.test_end(name, + 'SKIP', + message=test['disabled']) + self.todo += 1 + + def run_tests(self, tests): + start_time = time.time() + self._initialize_test_run(tests) + + if self.marionette is None: + self.marionette = self.driverclass(**self._build_kwargs()) + self.logger.info("Profile path is %s" % self.marionette.profile_path) + + if len(self.fixture_servers) == 0 or \ + any(not server.is_alive for _, server in self.fixture_servers): + self.logger.info("Starting fixture servers") + self.fixture_servers = self.start_fixture_servers() + for url in iter_url(self.fixture_servers): + self.logger.info("Fixture server listening on %s" % url) + + # backwards compatibility + self.marionette.baseurl = serve.where_is("/") + + self._add_tests(tests) + + device_info = None + if self.marionette.instance and self.emulator: + try: + device_info = self.marionette.instance.runner.device.dm.getInfo() + except Exception: + self.logger.warning('Could not get device info', exc_info=True) + + appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False) + self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled")) + if self.e10s != appinfo_e10s: + message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does " + "not match browser appinfo") + self.cleanup() + raise AssertionError(message_e10s) + + self.logger.suite_start(self.tests, + version_info=self.version_info, + device_info=device_info) + + self._log_skipped_tests() + + interrupted = None + try: + counter = self.repeat + while counter >= 0: + round_num = self.repeat - counter + if round_num > 0: + self.logger.info('\nREPEAT {}\n-------'.format(round_num)) + self.run_test_sets() + counter -= 1 + except KeyboardInterrupt: + # in case of KeyboardInterrupt during the test execution + # we want to display current test results. + # so we keep the exception to raise it later. + interrupted = sys.exc_info() + except: + # For any other exception we return immediately and have to + # cleanup running processes + self.cleanup() + raise + + try: + self._print_summary(tests) + self.record_crash() + self.elapsedtime = time.time() - start_time + + for run_tests in self.mixin_run_tests: + run_tests(tests) + if self.shuffle: + self.logger.info("Using shuffle seed: %d" % self.shuffle_seed) + + self.logger.suite_end() + except: + # raise only the exception if we were not interrupted + if not interrupted: + raise + finally: + self.cleanup() + + # reraise previous interruption now + if interrupted: + raise interrupted[0], interrupted[1], interrupted[2] + + def _print_summary(self, tests): + self.logger.info('\nSUMMARY\n-------') + self.logger.info('passed: {}'.format(self.passed)) + if self.unexpected_successes == 0: + self.logger.info('failed: {}'.format(self.failed)) + else: + self.logger.info( + 'failed: {0} (unexpected sucesses: {1})'.format(self.failed, + self.unexpected_successes)) + if self.skipped == 0: + self.logger.info('todo: {}'.format(self.todo)) + else: + self.logger.info('todo: {0} (skipped: {1})'.format(self.todo, self.skipped)) + + if self.failed > 0: + self.logger.info('\nFAILED TESTS\n-------') + for failed_test in self.failures: + self.logger.info('{}'.format(failed_test[0])) + + def start_fixture_servers(self): + root = self.server_root or os.path.join(os.path.dirname(here), "www") + if self.appName == "fennec": + return serve.start(root, host=moznetwork.get_ip()) + else: + return serve.start(root) + + def add_test(self, test, expected='pass'): + filepath = os.path.abspath(test) + + if os.path.isdir(filepath): + for root, dirs, files in os.walk(filepath): + for filename in files: + if filename.endswith('.ini'): + msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'." + " See --help for details.") + relpath = os.path.relpath(os.path.join(root, filename), filepath) + self.logger.warning(msg_tmpl.format(relpath, filepath)) + elif self._is_filename_valid(filename): + test_file = os.path.join(root, filename) + self.add_test(test_file) + return + + file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] + + if file_ext == '.ini': + manifest = TestManifest() + manifest.read(filepath) + + json_path = update_mozinfo(filepath) + self.logger.info("mozinfo updated from: {}".format(json_path)) + self.logger.info("mozinfo is: {}".format(mozinfo.info)) + + filters = [] + if self.test_tags: + filters.append(tags(self.test_tags)) + + values = { + "appname": self.appName, + "e10s": self.e10s, + "manage_instance": self.marionette.instance is not None, + } + values.update(mozinfo.info) + + manifest_tests = manifest.active_tests(exists=False, + disabled=True, + filters=filters, + **values) + if len(manifest_tests) == 0: + self.logger.error("No tests to run using specified " + "combination of filters: {}".format( + manifest.fmt_filters())) + + target_tests = [] + for test in manifest_tests: + if test.get('disabled'): + self.manifest_skipped_tests.append(test) + else: + target_tests.append(test) + + for i in target_tests: + if not os.path.exists(i["path"]): + raise IOError("test file: {} does not exist".format(i["path"])) + + file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1] + + self.add_test(i["path"], i["expected"]) + return + + self.tests.append({'filepath': filepath, 'expected': expected}) + + def run_test(self, filepath, expected): + testloader = unittest.TestLoader() + suite = unittest.TestSuite() + self.test_kwargs['expected'] = expected + mod_name = os.path.splitext(os.path.split(filepath)[-1])[0] + for handler in self.test_handlers: + if handler.match(os.path.basename(filepath)): + handler.add_tests_to_suite(mod_name, + filepath, + suite, + testloader, + self.marionette, + self.fixtures, + self.testvars, + **self.test_kwargs) + break + + if suite.countTestCases(): + runner = self.textrunnerclass(logger=self.logger, + marionette=self.marionette, + capabilities=self.capabilities, + result_callbacks=self.result_callbacks) + + results = runner.run(suite) + self.results.append(results) + + self.failed += len(results.failures) + len(results.errors) + if hasattr(results, 'skipped'): + self.skipped += len(results.skipped) + self.todo += len(results.skipped) + self.passed += results.passed + for failure in results.failures + results.errors: + self.failures.append((results.getInfo(failure), failure.output, + 'TEST-UNEXPECTED-FAIL')) + if hasattr(results, 'unexpectedSuccesses'): + self.failed += len(results.unexpectedSuccesses) + self.unexpected_successes += len(results.unexpectedSuccesses) + for failure in results.unexpectedSuccesses: + self.failures.append((results.getInfo(failure), failure.output, + 'TEST-UNEXPECTED-PASS')) + if hasattr(results, 'expectedFailures'): + self.todo += len(results.expectedFailures) + + self.mixin_run_tests = [] + for result in self.results: + result.result_modifiers = [] + + def run_test_set(self, tests): + if self.shuffle: + random.seed(self.shuffle_seed) + random.shuffle(tests) + + for test in tests: + self.run_test(test['filepath'], test['expected']) + if self.record_crash(): + break + + def run_test_sets(self): + if len(self.tests) < 1: + raise Exception('There are no tests to run.') + elif self.total_chunks > len(self.tests): + raise ValueError('Total number of chunks must be between 1 and {}.' + .format(len(self.tests))) + if self.total_chunks > 1: + chunks = [[] for i in range(self.total_chunks)] + for i, test in enumerate(self.tests): + target_chunk = i % self.total_chunks + chunks[target_chunk].append(test) + + self.logger.info('Running chunk {0} of {1} ({2} tests selected from a ' + 'total of {3})'.format(self.this_chunk, self.total_chunks, + len(chunks[self.this_chunk - 1]), + len(self.tests))) + self.tests = chunks[self.this_chunk - 1] + + self.run_test_set(self.tests) + + def cleanup(self): + for proc in iter_proc(self.fixture_servers): + proc.stop() + proc.kill() + self.fixture_servers = {} + + if hasattr(self, 'marionette') and self.marionette: + if self.marionette.instance is not None: + self.marionette.instance.close() + self.marionette.instance = None + + self.marionette.cleanup() + self.marionette = None + + __del__ = cleanup |