summaryrefslogtreecommitdiffstats
path: root/testing/mozharness/scripts/marionette.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/mozharness/scripts/marionette.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/mozharness/scripts/marionette.py')
-rwxr-xr-xtesting/mozharness/scripts/marionette.py358
1 files changed, 358 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py
new file mode 100755
index 000000000..b7f9c2765
--- /dev/null
+++ b/testing/mozharness/scripts/marionette.py
@@ -0,0 +1,358 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import copy
+import os
+import re
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import TarErrorList
+from mozharness.base.log import INFO, ERROR, WARNING
+from mozharness.base.script import PreScriptAction
+from mozharness.base.transfer import TransferMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.testing.errors import LogcatErrorList
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+
+# TODO: we could remove emulator specific code after B2G ICS emulator buildbot
+# builds is turned off, Bug 1209180.
+
+
+class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMixin):
+ config_options = [[
+ ["--application"],
+ {"action": "store",
+ "dest": "application",
+ "default": None,
+ "help": "application name of binary"
+ }
+ ], [
+ ["--app-arg"],
+ {"action": "store",
+ "dest": "app_arg",
+ "default": None,
+ "help": "Optional command-line argument to pass to the browser"
+ }
+ ], [
+ ["--marionette-address"],
+ {"action": "store",
+ "dest": "marionette_address",
+ "default": None,
+ "help": "The host:port of the Marionette server running inside Gecko. Unused for emulator testing",
+ }
+ ], [
+ ["--emulator"],
+ {"action": "store",
+ "type": "choice",
+ "choices": ['arm', 'x86'],
+ "dest": "emulator",
+ "default": None,
+ "help": "Use an emulator for testing",
+ }
+ ], [
+ ["--test-manifest"],
+ {"action": "store",
+ "dest": "test_manifest",
+ "default": "unit-tests.ini",
+ "help": "Path to test manifest to run relative to the Marionette "
+ "tests directory",
+ }
+ ], [
+ ["--total-chunks"],
+ {"action": "store",
+ "dest": "total_chunks",
+ "help": "Number of total chunks",
+ }
+ ], [
+ ["--this-chunk"],
+ {"action": "store",
+ "dest": "this_chunk",
+ "help": "Number of this chunk",
+ }
+ ], [
+ ["--e10s"],
+ {"action": "store_true",
+ "dest": "e10s",
+ "default": False,
+ "help": "Run tests with multiple processes. (Desktop builds only)",
+ }
+ ], [
+ ["--allow-software-gl-layers"],
+ {"action": "store_true",
+ "dest": "allow_software_gl_layers",
+ "default": False,
+ "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."
+ }
+ ]] + copy.deepcopy(testing_config_options) \
+ + copy.deepcopy(blobupload_config_options)
+
+ error_list = [
+ {'substr': 'FAILED (errors=', 'level': WARNING},
+ {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
+ {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
+ {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
+ {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''), 'level': ERROR},
+ {'regex': re.compile(r'''(\b((?!Marionette|TestMarionette|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidResponse|Javascript|Timeout|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)\w*)Exception)'''), 'level': ERROR},
+ ]
+
+ repos = []
+
+ def __init__(self, require_config_file=False):
+ super(MarionetteTest, self).__init__(
+ config_options=self.config_options,
+ all_actions=['clobber',
+ 'read-buildbot-config',
+ 'pull',
+ 'download-and-extract',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests'],
+ default_actions=['clobber',
+ 'pull',
+ 'download-and-extract',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests'],
+ require_config_file=require_config_file,
+ config={'require_test_zip': True})
+
+ # these are necessary since self.config is read only
+ c = self.config
+ self.installer_url = c.get('installer_url')
+ self.installer_path = c.get('installer_path')
+ self.binary_path = c.get('binary_path')
+ self.test_url = c.get('test_url')
+ self.test_packages_url = c.get('test_packages_url')
+
+ if c.get('structured_output'):
+ self.parser_class = StructuredOutputParser
+ else:
+ self.parser_class = TestSummaryOutputParserHelper
+
+ def _pre_config_lock(self, rw_config):
+ super(MarionetteTest, self)._pre_config_lock(rw_config)
+ if not self.config.get('emulator') and not self.config.get('marionette_address'):
+ self.fatal("You need to specify a --marionette-address for non-emulator tests! (Try --marionette-address localhost:2828 )")
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(MarionetteTest, self).query_abs_dirs()
+ dirs = {}
+ dirs['abs_test_install_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'], 'tests')
+ dirs['abs_marionette_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], 'marionette', 'harness', 'marionette_harness')
+ dirs['abs_marionette_tests_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], 'marionette', 'tests', 'testing',
+ 'marionette', 'harness', 'marionette_harness', 'tests')
+ dirs['abs_gecko_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'], 'gecko')
+ dirs['abs_emulator_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'], 'emulator')
+
+ dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ @PreScriptAction('create-virtualenv')
+ def _configure_marionette_virtualenv(self, action):
+ dirs = self.query_abs_dirs()
+ requirements = os.path.join(dirs['abs_test_install_dir'],
+ 'config',
+ 'marionette_requirements.txt')
+ if os.access(requirements, os.F_OK):
+ self.register_virtualenv_module(requirements=[requirements],
+ two_pass=True)
+ else:
+ # XXX Bug 879765: Dependent modules need to be listed before parent
+ # modules, otherwise they will get installed from the pypi server.
+ # XXX Bug 908356: This block can be removed as soon as the
+ # in-tree requirements files propagate to all active trees.
+ mozbase_dir = os.path.join('tests', 'mozbase')
+ self.register_virtualenv_module(
+ 'manifestparser', os.path.join(mozbase_dir, 'manifestdestiny'))
+ for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
+ 'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile',
+ 'mozprocess', 'mozrunner'):
+ self.register_virtualenv_module(
+ m, os.path.join(mozbase_dir, m))
+
+ self.register_virtualenv_module(
+ 'marionette', os.path.join('tests', 'marionette'))
+
+ def _get_options_group(self, is_emulator):
+ """
+ Determine which in tree options group to use and return the
+ appropriate key.
+ """
+ platform = 'emulator' if is_emulator else 'desktop'
+ # Currently running marionette on an emulator means webapi
+ # tests. This method will need to change if this does.
+ testsuite = 'webapi' if is_emulator else 'marionette'
+ return '{}_{}'.format(testsuite, platform)
+
+ def download_and_extract(self):
+ super(MarionetteTest, self).download_and_extract()
+
+ if self.config.get('emulator'):
+ dirs = self.query_abs_dirs()
+
+ self.mkdir_p(dirs['abs_emulator_dir'])
+ tar = self.query_exe('tar', return_type='list')
+ self.run_command(tar + ['zxf', self.installer_path],
+ cwd=dirs['abs_emulator_dir'],
+ error_list=TarErrorList,
+ halt_on_failure=True, fatal_exit_code=3)
+
+ def install(self):
+ if self.config.get('emulator'):
+ self.info("Emulator tests; skipping.")
+ else:
+ super(MarionetteTest, self).install()
+
+ def run_tests(self):
+ """
+ Run the Marionette tests
+ """
+ dirs = self.query_abs_dirs()
+
+ raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
+ 'marionette_raw.log')
+ error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
+ 'marionette_errorsummary.log')
+ html_report_file = os.path.join(dirs['abs_blob_upload_dir'],
+ 'report.html')
+
+ config_fmt_args = {
+ # emulator builds require a longer timeout
+ 'timeout': 60000 if self.config.get('emulator') else 10000,
+ 'profile': os.path.join(dirs['abs_work_dir'], 'profile'),
+ 'xml_output': os.path.join(dirs['abs_work_dir'], 'output.xml'),
+ 'html_output': os.path.join(dirs['abs_blob_upload_dir'], 'output.html'),
+ 'logcat_dir': dirs['abs_work_dir'],
+ 'emulator': 'arm',
+ 'symbols_path': self.symbols_path,
+ 'binary': self.binary_path,
+ 'address': self.config.get('marionette_address'),
+ 'raw_log_file': raw_log_file,
+ 'error_summary_file': error_summary_file,
+ 'html_report_file': html_report_file,
+ 'gecko_log': dirs["abs_blob_upload_dir"],
+ 'this_chunk': self.config.get('this_chunk', 1),
+ 'total_chunks': self.config.get('total_chunks', 1)
+ }
+
+ self.info("The emulator type: %s" % config_fmt_args["emulator"])
+ # build the marionette command arguments
+ python = self.query_python_path('python')
+
+ cmd = [python, '-u', os.path.join(dirs['abs_marionette_dir'],
+ 'runtests.py')]
+
+ manifest = os.path.join(dirs['abs_marionette_tests_dir'],
+ self.config['test_manifest'])
+
+ if self.config.get('app_arg'):
+ config_fmt_args['app_arg'] = self.config['app_arg']
+
+ if not self.config['e10s']:
+ cmd.append('--disable-e10s')
+
+ cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
+ 'gecko.log'))
+
+ if self.config.get("structured_output"):
+ cmd.append("--log-raw=-")
+
+ options_group = self._get_options_group(self.config.get('emulator'))
+
+ if options_group not in self.config["suite_definitions"]:
+ self.fatal("%s is not defined in the config!" % options_group)
+
+ for s in self.config["suite_definitions"][options_group]["options"]:
+ cmd.append(s % config_fmt_args)
+
+ if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
+ # Make sure that the logging directory exists
+ self.fatal("Could not create blobber upload directory")
+
+ cmd.append(manifest)
+
+ try_options, try_tests = self.try_args("marionette")
+ cmd.extend(self.query_tests_args(try_tests,
+ str_format_values=config_fmt_args))
+
+ env = {}
+ if self.query_minidump_stackwalk():
+ env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
+ env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+ env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+
+ if self.config['allow_software_gl_layers']:
+ env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
+
+ if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+ self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+ env = self.query_env(partial_env=env)
+
+ marionette_parser = self.parser_class(config=self.config,
+ log_obj=self.log_obj,
+ error_list=self.error_list,
+ strict=False)
+ return_code = self.run_command(cmd, env=env,
+ output_timeout=1000,
+ output_parser=marionette_parser)
+ level = INFO
+ tbpl_status, log_level = marionette_parser.evaluate_parser(
+ return_code=return_code)
+ marionette_parser.append_tinderboxprint_line("marionette")
+
+ qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
+ if os.path.isfile(qemu):
+ self.copyfile(qemu, os.path.join(dirs['abs_blob_upload_dir'],
+ 'qemu.log'))
+
+ # dump logcat output if there were failures
+ if self.config.get('emulator'):
+ if marionette_parser.failed != "0" or 'T-FAIL' in marionette_parser.tsummary:
+ logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
+ if os.access(logcat, os.F_OK):
+ self.info('dumping logcat')
+ self.run_command(['cat', logcat], error_list=LogcatErrorList)
+ else:
+ self.info('no logcat file found')
+ else:
+ # .. or gecko.log if it exists
+ gecko_log = os.path.join(self.config['base_work_dir'], 'gecko.log')
+ if os.access(gecko_log, os.F_OK):
+ self.info('dumping gecko.log')
+ self.run_command(['cat', gecko_log])
+ self.rmtree(gecko_log)
+ else:
+ self.info('gecko.log not found')
+
+ marionette_parser.print_summary('marionette')
+
+ self.log("Marionette exited with return code %s: %s" % (return_code, tbpl_status),
+ level=level)
+ self.buildbot_status(tbpl_status)
+
+
+if __name__ == '__main__':
+ marionetteTest = MarionetteTest()
+ marionetteTest.run_and_exit()