summaryrefslogtreecommitdiffstats
path: root/testing/mochitest/mach_commands.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/mochitest/mach_commands.py')
-rw-r--r--testing/mochitest/mach_commands.py567
1 files changed, 567 insertions, 0 deletions
diff --git a/testing/mochitest/mach_commands.py b/testing/mochitest/mach_commands.py
new file mode 100644
index 000000000..fb261ec82
--- /dev/null
+++ b/testing/mochitest/mach_commands.py
@@ -0,0 +1,567 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+from argparse import Namespace
+from collections import defaultdict
+from itertools import chain
+import logging
+import os
+import sys
+import warnings
+
+from mozbuild.base import (
+ MachCommandBase,
+ MachCommandConditions as conditions,
+ MozbuildObject,
+)
+
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+ENG_BUILD_REQUIRED = '''
+The mochitest command requires an engineering build. It may be the case that
+VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
+
+ $ VARIANT=eng ./build.sh
+
+There should be an app called 'test-container.gaiamobile.org' located in
+{}.
+'''.lstrip()
+
+SUPPORTED_TESTS_NOT_FOUND = '''
+The mochitest command could not find any supported tests to run! The
+following flavors and subsuites were found, but are either not supported on
+{} builds, or were excluded on the command line:
+
+{}
+
+Double check the command line you used, and make sure you are running in
+context of the proper build. To switch build contexts, either run |mach|
+from the appropriate objdir, or export the correct mozconfig:
+
+ $ export MOZCONFIG=path/to/mozconfig
+'''.lstrip()
+
+TESTS_NOT_FOUND = '''
+The mochitest command could not find any mochitests under the following
+test path(s):
+
+{}
+
+Please check spelling and make sure there are mochitests living there.
+'''.lstrip()
+
+ROBOCOP_TESTS_NOT_FOUND = '''
+The robocop command could not find any tests under the following
+test path(s):
+
+{}
+
+Please check spelling and make sure the named tests exist.
+'''.lstrip()
+
+NOW_RUNNING = '''
+######
+### Now running mochitest-{}.
+######
+'''
+
+
+# Maps test flavors to data needed to run them
+ALL_FLAVORS = {
+ 'mochitest': {
+ 'suite': 'plain',
+ 'aliases': ('plain', 'mochitest'),
+ 'enabled_apps': ('firefox', 'android'),
+ 'extra_args': {
+ 'flavor': 'plain',
+ }
+ },
+ 'chrome': {
+ 'suite': 'chrome',
+ 'aliases': ('chrome', 'mochitest-chrome'),
+ 'enabled_apps': ('firefox', 'android'),
+ 'extra_args': {
+ 'flavor': 'chrome',
+ }
+ },
+ 'browser-chrome': {
+ 'suite': 'browser',
+ 'aliases': ('browser', 'browser-chrome', 'mochitest-browser-chrome', 'bc'),
+ 'enabled_apps': ('firefox',),
+ 'extra_args': {
+ 'flavor': 'browser',
+ }
+ },
+ 'jetpack-package': {
+ 'suite': 'jetpack-package',
+ 'aliases': ('jetpack-package', 'mochitest-jetpack-package', 'jpp'),
+ 'enabled_apps': ('firefox',),
+ 'extra_args': {
+ 'flavor': 'jetpack-package',
+ }
+ },
+ 'jetpack-addon': {
+ 'suite': 'jetpack-addon',
+ 'aliases': ('jetpack-addon', 'mochitest-jetpack-addon', 'jpa'),
+ 'enabled_apps': ('firefox',),
+ 'extra_args': {
+ 'flavor': 'jetpack-addon',
+ }
+ },
+ 'a11y': {
+ 'suite': 'a11y',
+ 'aliases': ('a11y', 'mochitest-a11y', 'accessibility'),
+ 'enabled_apps': ('firefox',),
+ 'extra_args': {
+ 'flavor': 'a11y',
+ }
+ },
+}
+
+SUPPORTED_APPS = ['firefox', 'android']
+SUPPORTED_FLAVORS = list(chain.from_iterable([f['aliases'] for f in ALL_FLAVORS.values()]))
+CANONICAL_FLAVORS = sorted([f['aliases'][0] for f in ALL_FLAVORS.values()])
+
+parser = None
+
+
+class MochitestRunner(MozbuildObject):
+
+ """Easily run mochitests.
+
+ This currently contains just the basics for running mochitests. We may want
+ to hook up result parsing, etc.
+ """
+
+ def __init__(self, *args, **kwargs):
+ MozbuildObject.__init__(self, *args, **kwargs)
+
+ # TODO Bug 794506 remove once mach integrates with virtualenv.
+ build_path = os.path.join(self.topobjdir, 'build')
+ if build_path not in sys.path:
+ sys.path.append(build_path)
+
+ self.tests_dir = os.path.join(self.topobjdir, '_tests')
+ self.mochitest_dir = os.path.join(
+ self.tests_dir,
+ 'testing',
+ 'mochitest')
+ self.bin_dir = os.path.join(self.topobjdir, 'dist', 'bin')
+
+ def resolve_tests(self, test_paths, test_objects=None, cwd=None):
+ if test_objects:
+ return test_objects
+
+ from mozbuild.testing import TestResolver
+ resolver = self._spawn(TestResolver)
+ tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
+ return tests
+
+ def run_desktop_test(self, context, tests=None, suite=None, **kwargs):
+ """Runs a mochitest.
+
+ suite is the type of mochitest to run. It can be one of ('plain',
+ 'chrome', 'browser', 'a11y', 'jetpack-package', 'jetpack-addon').
+ """
+ # runtests.py is ambiguous, so we load the file/module manually.
+ if 'mochitest' not in sys.modules:
+ import imp
+ path = os.path.join(self.mochitest_dir, 'runtests.py')
+ with open(path, 'r') as fh:
+ imp.load_module('mochitest', fh, path,
+ ('.py', 'r', imp.PY_SOURCE))
+
+ import mochitest
+
+ # This is required to make other components happy. Sad, isn't it?
+ os.chdir(self.topobjdir)
+
+ # Automation installs its own stream handler to stdout. Since we want
+ # all logging to go through us, we just remove their handler.
+ remove_handlers = [l for l in logging.getLogger().handlers
+ if isinstance(l, logging.StreamHandler)]
+ for handler in remove_handlers:
+ logging.getLogger().removeHandler(handler)
+
+ options = Namespace(**kwargs)
+
+ from manifestparser import TestManifest
+ if tests and not options.manifestFile:
+ manifest = TestManifest()
+ manifest.tests.extend(tests)
+ options.manifestFile = manifest
+
+ # When developing mochitest-plain tests, it's often useful to be able to
+ # refresh the page to pick up modifications. Therefore leave the browser
+ # open if only running a single mochitest-plain test. This behaviour can
+ # be overridden by passing in --keep-open=false.
+ if len(tests) == 1 and options.keep_open is None and suite == 'plain':
+ options.keep_open = True
+
+ # We need this to enable colorization of output.
+ self.log_manager.enable_unstructured()
+ result = mochitest.run_test_harness(parser, options)
+ self.log_manager.disable_unstructured()
+ return result
+
+ def run_android_test(self, context, tests, suite=None, **kwargs):
+ host_ret = verify_host_bin()
+ if host_ret != 0:
+ return host_ret
+
+ import imp
+ path = os.path.join(self.mochitest_dir, 'runtestsremote.py')
+ with open(path, 'r') as fh:
+ imp.load_module('runtestsremote', fh, path,
+ ('.py', 'r', imp.PY_SOURCE))
+ import runtestsremote
+
+ options = Namespace(**kwargs)
+
+ from manifestparser import TestManifest
+ if tests and not options.manifestFile:
+ manifest = TestManifest()
+ manifest.tests.extend(tests)
+ options.manifestFile = manifest
+
+ return runtestsremote.run_test_harness(parser, options)
+
+ def run_robocop_test(self, context, tests, suite=None, **kwargs):
+ host_ret = verify_host_bin()
+ if host_ret != 0:
+ return host_ret
+
+ import imp
+ path = os.path.join(self.mochitest_dir, 'runrobocop.py')
+ with open(path, 'r') as fh:
+ imp.load_module('runrobocop', fh, path,
+ ('.py', 'r', imp.PY_SOURCE))
+ import runrobocop
+
+ options = Namespace(**kwargs)
+
+ from manifestparser import TestManifest
+ if tests and not options.manifestFile:
+ manifest = TestManifest()
+ manifest.tests.extend(tests)
+ options.manifestFile = manifest
+
+ return runrobocop.run_test_harness(parser, options)
+
+# parser
+
+
+def setup_argument_parser():
+ build_obj = MozbuildObject.from_environment(cwd=here)
+
+ build_path = os.path.join(build_obj.topobjdir, 'build')
+ if build_path not in sys.path:
+ sys.path.append(build_path)
+
+ mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest')
+
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+
+ import imp
+ path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py')
+ with open(path, 'r') as fh:
+ imp.load_module('mochitest', fh, path,
+ ('.py', 'r', imp.PY_SOURCE))
+
+ from mochitest_options import MochitestArgumentParser
+
+ if conditions.is_android(build_obj):
+ # On Android, check for a connected device (and offer to start an
+ # emulator if appropriate) before running tests. This check must
+ # be done in this admittedly awkward place because
+ # MochitestArgumentParser initialization fails if no device is found.
+ from mozrunner.devices.android_device import verify_android_device
+ verify_android_device(build_obj, install=True, xre=True)
+
+ global parser
+ parser = MochitestArgumentParser()
+ return parser
+
+
+# condition filters
+
+def is_buildapp_in(*apps):
+ def is_buildapp_supported(cls):
+ for a in apps:
+ c = getattr(conditions, 'is_{}'.format(a), None)
+ if c and c(cls):
+ return True
+ return False
+
+ is_buildapp_supported.__doc__ = 'Must have a {} build.'.format(
+ ' or '.join(apps))
+ return is_buildapp_supported
+
+
+def verify_host_bin():
+ # validate MOZ_HOST_BIN environment variables for Android tests
+ MOZ_HOST_BIN = os.environ.get('MOZ_HOST_BIN')
+ if not MOZ_HOST_BIN:
+ print('environment variable MOZ_HOST_BIN must be set to a directory containing host '
+ 'xpcshell')
+ return 1
+ elif not os.path.isdir(MOZ_HOST_BIN):
+ print('$MOZ_HOST_BIN does not specify a directory')
+ return 1
+ elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, 'xpcshell')):
+ print('$MOZ_HOST_BIN/xpcshell does not exist')
+ return 1
+ return 0
+
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+ @Command('mochitest', category='testing',
+ conditions=[is_buildapp_in(*SUPPORTED_APPS)],
+ description='Run any flavor of mochitest (integration test).',
+ parser=setup_argument_parser)
+ @CommandArgument('-f', '--flavor',
+ metavar='{{{}}}'.format(', '.join(CANONICAL_FLAVORS)),
+ choices=SUPPORTED_FLAVORS,
+ help='Only run tests of this flavor.')
+ def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
+ buildapp = None
+ for app in SUPPORTED_APPS:
+ if is_buildapp_in(app)(self):
+ buildapp = app
+ break
+
+ flavors = None
+ if flavor:
+ for fname, fobj in ALL_FLAVORS.iteritems():
+ if flavor in fobj['aliases']:
+ if buildapp not in fobj['enabled_apps']:
+ continue
+ flavors = [fname]
+ break
+ else:
+ flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]
+
+ from mozbuild.controller.building import BuildDriver
+ self._ensure_state_subdir_exists('.')
+
+ test_paths = kwargs['test_paths']
+ kwargs['test_paths'] = []
+
+ mochitest = self._spawn(MochitestRunner)
+ tests = []
+ if resolve_tests:
+ tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)
+
+ driver = self._spawn(BuildDriver)
+ driver.install_tests(tests)
+
+ subsuite = kwargs.get('subsuite')
+ if subsuite == 'default':
+ kwargs['subsuite'] = None
+
+ suites = defaultdict(list)
+ unsupported = set()
+ for test in tests:
+ # Filter out non-mochitests and unsupported flavors.
+ if test['flavor'] not in ALL_FLAVORS:
+ continue
+
+ key = (test['flavor'], test.get('subsuite', ''))
+ if test['flavor'] not in flavors:
+ unsupported.add(key)
+ continue
+
+ if subsuite == 'default':
+ # "--subsuite default" means only run tests that don't have a subsuite
+ if test.get('subsuite'):
+ unsupported.add(key)
+ continue
+ elif subsuite and test.get('subsuite', '') != subsuite:
+ unsupported.add(key)
+ continue
+
+ suites[key].append(test)
+
+ if ('mochitest', 'media') in suites:
+ req = os.path.join('testing', 'tools', 'websocketprocessbridge',
+ 'websocketprocessbridge_requirements.txt')
+ self.virtualenv_manager.activate()
+ self.virtualenv_manager.install_pip_requirements(req, require_hashes=False)
+
+ # sys.executable is used to start the websocketprocessbridge, though for some
+ # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
+ sys.executable = self.virtualenv_manager.python_path
+
+ # This is a hack to introduce an option in mach to not send
+ # filtered tests to the mochitest harness. Mochitest harness will read
+ # the master manifest in that case.
+ if not resolve_tests:
+ for flavor in flavors:
+ key = (flavor, kwargs.get('subsuite'))
+ suites[key] = []
+
+ if not suites:
+ # Make it very clear why no tests were found
+ if not unsupported:
+ print(TESTS_NOT_FOUND.format('\n'.join(
+ sorted(list(test_paths or test_objects)))))
+ return 1
+
+ msg = []
+ for f, s in unsupported:
+ fobj = ALL_FLAVORS[f]
+ apps = fobj['enabled_apps']
+ name = fobj['aliases'][0]
+ if s:
+ name = '{} --subsuite {}'.format(name, s)
+
+ if buildapp not in apps:
+ reason = 'requires {}'.format(' or '.join(apps))
+ else:
+ reason = 'excluded by the command line'
+ msg.append(' mochitest -f {} ({})'.format(name, reason))
+ print(SUPPORTED_TESTS_NOT_FOUND.format(
+ buildapp, '\n'.join(sorted(msg))))
+ return 1
+
+ if buildapp == 'android':
+ from mozrunner.devices.android_device import grant_runtime_permissions
+ grant_runtime_permissions(self)
+ run_mochitest = mochitest.run_android_test
+ else:
+ run_mochitest = mochitest.run_desktop_test
+
+ overall = None
+ for (flavor, subsuite), tests in sorted(suites.items()):
+ fobj = ALL_FLAVORS[flavor]
+ msg = fobj['aliases'][0]
+ if subsuite:
+ msg = '{} with subsuite {}'.format(msg, subsuite)
+ print(NOW_RUNNING.format(msg))
+
+ harness_args = kwargs.copy()
+ harness_args['subsuite'] = subsuite
+ harness_args.update(fobj.get('extra_args', {}))
+
+ result = run_mochitest(
+ self._mach_context,
+ tests=tests,
+ suite=fobj['suite'],
+ **harness_args)
+
+ if result:
+ overall = result
+
+ # TODO consolidate summaries from all suites
+ return overall
+
+
+@CommandProvider
+class RobocopCommands(MachCommandBase):
+
+ @Command('robocop', category='testing',
+ conditions=[conditions.is_android],
+ description='Run a Robocop test.',
+ parser=setup_argument_parser)
+ @CommandArgument('--serve', default=False, action='store_true',
+ help='Run no tests but start the mochi.test web server '
+ 'and launch Fennec with a test profile.')
+ def run_robocop(self, serve=False, **kwargs):
+ if serve:
+ kwargs['autorun'] = False
+
+ if not kwargs.get('robocopIni'):
+ kwargs['robocopIni'] = os.path.join(self.topobjdir, '_tests', 'testing',
+ 'mochitest', 'robocop.ini')
+
+ if not kwargs.get('robocopApk'):
+ kwargs['robocopApk'] = os.path.join(self.topobjdir, 'mobile', 'android',
+ 'tests', 'browser', 'robocop',
+ 'robocop-debug.apk')
+
+ from mozbuild.controller.building import BuildDriver
+ self._ensure_state_subdir_exists('.')
+
+ test_paths = kwargs['test_paths']
+ kwargs['test_paths'] = []
+
+ from mozbuild.testing import TestResolver
+ resolver = self._spawn(TestResolver)
+ tests = list(resolver.resolve_tests(paths=test_paths, cwd=self._mach_context.cwd,
+ flavor='instrumentation', subsuite='robocop'))
+ driver = self._spawn(BuildDriver)
+ driver.install_tests(tests)
+
+ if len(tests) < 1:
+ print(ROBOCOP_TESTS_NOT_FOUND.format('\n'.join(
+ sorted(list(test_paths)))))
+ return 1
+
+ from mozrunner.devices.android_device import grant_runtime_permissions
+ grant_runtime_permissions(self)
+
+ mochitest = self._spawn(MochitestRunner)
+ return mochitest.run_robocop_test(self._mach_context, tests, 'robocop', **kwargs)
+
+
+# NOTE python/mach/mach/commands/commandinfo.py references this function
+# by name. If this function is renamed or removed, that file should
+# be updated accordingly as well.
+def REMOVED(cls):
+ """Command no longer exists! Use |mach mochitest| instead.
+
+ The |mach mochitest| command will automatically detect which flavors and
+ subsuites exist in a given directory. If desired, flavors and subsuites
+ can be restricted using `--flavor` and `--subsuite` respectively. E.g:
+
+ $ ./mach mochitest dom/indexedDB
+
+ will run all of the plain, chrome and browser-chrome mochitests in that
+ directory. To only run the plain mochitests:
+
+ $ ./mach mochitest -f plain dom/indexedDB
+ """
+ return False
+
+
+@CommandProvider
+class DeprecatedCommands(MachCommandBase):
+ @Command('mochitest-plain', category='testing', conditions=[REMOVED])
+ def mochitest_plain(self):
+ pass
+
+ @Command('mochitest-chrome', category='testing', conditions=[REMOVED])
+ def mochitest_chrome(self):
+ pass
+
+ @Command('mochitest-browser', category='testing', conditions=[REMOVED])
+ def mochitest_browser(self):
+ pass
+
+ @Command('mochitest-devtools', category='testing', conditions=[REMOVED])
+ def mochitest_devtools(self):
+ pass
+
+ @Command('mochitest-a11y', category='testing', conditions=[REMOVED])
+ def mochitest_a11y(self):
+ pass
+
+ @Command('jetpack-addon', category='testing', conditions=[REMOVED])
+ def jetpack_addon(self):
+ pass
+
+ @Command('jetpack-package', category='testing', conditions=[REMOVED])
+ def jetpack_package(self):
+ pass