diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/executors')
17 files changed, 2103 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/executors/__init__.py b/testing/web-platform/harness/wptrunner/executors/__init__.py new file mode 100644 index 000000000..11e0c4cd7 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/__init__.py @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from base import (executor_kwargs, + testharness_result_converter, + reftest_result_converter, + TestExecutor) diff --git a/testing/web-platform/harness/wptrunner/executors/base.py b/testing/web-platform/harness/wptrunner/executors/base.py new file mode 100644 index 000000000..f0ce16658 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/base.py @@ -0,0 +1,329 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import json +import os +import traceback +import urlparse +from abc import ABCMeta, abstractmethod + +from ..testrunner import Stop + +here = os.path.split(__file__)[0] + + +def executor_kwargs(test_type, server_config, cache_manager, **kwargs): + timeout_multiplier = kwargs["timeout_multiplier"] + if timeout_multiplier is None: + timeout_multiplier = 1 + + executor_kwargs = {"server_config": server_config, + "timeout_multiplier": timeout_multiplier, + "debug_info": kwargs["debug_info"]} + + if test_type == "reftest": + executor_kwargs["screenshot_cache"] = cache_manager.dict() + + return executor_kwargs + + +def strip_server(url): + """Remove the scheme and netloc from a url, leaving only the path and any query + or fragment. + + url - the url to strip + + e.g. http://example.org:8000/tests?id=1#2 becomes /tests?id=1#2""" + + url_parts = list(urlparse.urlsplit(url)) + url_parts[0] = "" + url_parts[1] = "" + return urlparse.urlunsplit(url_parts) + + +class TestharnessResultConverter(object): + harness_codes = {0: "OK", + 1: "ERROR", + 2: "TIMEOUT"} + + test_codes = {0: "PASS", + 1: "FAIL", + 2: "TIMEOUT", + 3: "NOTRUN"} + + def __call__(self, test, result): + """Convert a JSON result into a (TestResult, [SubtestResult]) tuple""" + result_url, status, message, stack, subtest_results = result + assert result_url == test.url, ("Got results from %s, expected %s" % + (result_url, test.url)) + harness_result = test.result_cls(self.harness_codes[status], message) + return (harness_result, + [test.subtest_result_cls(name, self.test_codes[status], message, stack) + for name, status, message, stack in subtest_results]) + + +testharness_result_converter = TestharnessResultConverter() + + +def reftest_result_converter(self, test, result): + return (test.result_cls(result["status"], result["message"], + extra=result.get("extra")), []) + + +def pytest_result_converter(self, test, data): + harness_data, subtest_data = data + + if subtest_data is None: + subtest_data = [] + + harness_result = test.result_cls(*harness_data) + subtest_results = [test.subtest_result_cls(*item) for item in subtest_data] + + return (harness_result, subtest_results) + + +class ExecutorException(Exception): + def __init__(self, status, message): + self.status = status + self.message = message + + +class TestExecutor(object): + __metaclass__ = ABCMeta + + test_type = None + convert_result = None + + def __init__(self, browser, server_config, timeout_multiplier=1, + debug_info=None): + """Abstract Base class for object that actually executes the tests in a + specific browser. Typically there will be a different TestExecutor + subclass for each test type and method of executing tests. + + :param browser: ExecutorBrowser instance providing properties of the + browser that will be tested. + :param server_config: Dictionary of wptserve server configuration of the + form stored in TestEnvironment.external_config + :param timeout_multiplier: Multiplier relative to base timeout to use + when setting test timeout. + """ + self.runner = None + self.browser = browser + self.server_config = server_config + self.timeout_multiplier = timeout_multiplier + self.debug_info = debug_info + self.last_environment = {"protocol": "http", + "prefs": {}} + self.protocol = None # This must be set in subclasses + + @property + def logger(self): + """StructuredLogger for this executor""" + if self.runner is not None: + return self.runner.logger + + def setup(self, runner): + """Run steps needed before tests can be started e.g. connecting to + browser instance + + :param runner: TestRunner instance that is going to run the tests""" + self.runner = runner + if self.protocol is not None: + self.protocol.setup(runner) + + def teardown(self): + """Run cleanup steps after tests have finished""" + if self.protocol is not None: + self.protocol.teardown() + + def run_test(self, test): + """Run a particular test. + + :param test: The test to run""" + if test.environment != self.last_environment: + self.on_environment_change(test.environment) + + try: + result = self.do_test(test) + except Exception as e: + result = self.result_from_exception(test, e) + + if result is Stop: + return result + + # log result of parent test + if result[0].status == "ERROR": + self.logger.debug(result[0].message) + + self.last_environment = test.environment + + self.runner.send_message("test_ended", test, result) + + def server_url(self, protocol): + return "%s://%s:%s" % (protocol, + self.server_config["host"], + self.server_config["ports"][protocol][0]) + + def test_url(self, test): + return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url) + + @abstractmethod + def do_test(self, test): + """Test-type and protocol specific implementation of running a + specific test. + + :param test: The test to run.""" + pass + + def on_environment_change(self, new_environment): + pass + + def result_from_exception(self, test, e): + if hasattr(e, "status") and e.status in test.result_cls.statuses: + status = e.status + else: + status = "ERROR" + message = unicode(getattr(e, "message", "")) + if message: + message += "\n" + message += traceback.format_exc(e) + return test.result_cls(status, message), [] + + +class TestharnessExecutor(TestExecutor): + convert_result = testharness_result_converter + + +class RefTestExecutor(TestExecutor): + convert_result = reftest_result_converter + + def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None, + debug_info=None): + TestExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + + self.screenshot_cache = screenshot_cache + + +class RefTestImplementation(object): + def __init__(self, executor): + self.timeout_multiplier = executor.timeout_multiplier + self.executor = executor + # Cache of url:(screenshot hash, screenshot). Typically the + # screenshot is None, but we set this value if a test fails + # and the screenshot was taken from the cache so that we may + # retrieve the screenshot from the cache directly in the future + self.screenshot_cache = self.executor.screenshot_cache + self.message = None + + @property + def logger(self): + return self.executor.logger + + def get_hash(self, test, viewport_size, dpi): + timeout = test.timeout * self.timeout_multiplier + key = (test.url, viewport_size, dpi) + + if key not in self.screenshot_cache: + success, data = self.executor.screenshot(test, viewport_size, dpi) + + if not success: + return False, data + + screenshot = data + hash_value = hashlib.sha1(screenshot).hexdigest() + + self.screenshot_cache[key] = (hash_value, None) + + rv = (hash_value, screenshot) + else: + rv = self.screenshot_cache[key] + + self.message.append("%s %s" % (test.url, rv[0])) + return True, rv + + def is_pass(self, lhs_hash, rhs_hash, relation): + assert relation in ("==", "!=") + self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash)) + return ((relation == "==" and lhs_hash == rhs_hash) or + (relation == "!=" and lhs_hash != rhs_hash)) + + def run_test(self, test): + viewport_size = test.viewport_size + dpi = test.dpi + self.message = [] + + # Depth-first search of reference tree, with the goal + # of reachings a leaf node with only pass results + + stack = list(((test, item[0]), item[1]) for item in reversed(test.references)) + while stack: + hashes = [None, None] + screenshots = [None, None] + + nodes, relation = stack.pop() + + for i, node in enumerate(nodes): + success, data = self.get_hash(node, viewport_size, dpi) + if success is False: + return {"status": data[0], "message": data[1]} + + hashes[i], screenshots[i] = data + + if self.is_pass(hashes[0], hashes[1], relation): + if nodes[1].references: + stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references))) + else: + # We passed + return {"status":"PASS", "message": None} + + # We failed, so construct a failure message + + for i, (node, screenshot) in enumerate(zip(nodes, screenshots)): + if screenshot is None: + success, screenshot = self.retake_screenshot(node, viewport_size, dpi) + if success: + screenshots[i] = screenshot + + log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation, + {"url": nodes[1].url, "screenshot": screenshots[1]}] + + return {"status": "FAIL", + "message": "\n".join(self.message), + "extra": {"reftest_screenshots": log_data}} + + def retake_screenshot(self, node, viewport_size, dpi): + success, data = self.executor.screenshot(node, viewport_size, dpi) + if not success: + return False, data + + key = (node.url, viewport_size, dpi) + hash_val, _ = self.screenshot_cache[key] + self.screenshot_cache[key] = hash_val, data + return True, data + + +class WdspecExecutor(TestExecutor): + convert_result = pytest_result_converter + + +class Protocol(object): + def __init__(self, executor, browser): + self.executor = executor + self.browser = browser + + @property + def logger(self): + return self.executor.logger + + def setup(self, runner): + pass + + def teardown(self): + pass + + def wait(self): + pass diff --git a/testing/web-platform/harness/wptrunner/executors/executormarionette.py b/testing/web-platform/harness/wptrunner/executors/executormarionette.py new file mode 100644 index 000000000..e212c2945 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py @@ -0,0 +1,595 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import httplib +import os +import socket +import threading +import time +import traceback +import urlparse +import uuid +from collections import defaultdict + +from ..wpttest import WdspecResult, WdspecSubtestResult + +errors = None +marionette = None +webdriver = None + +here = os.path.join(os.path.split(__file__)[0]) + +from . import pytestrunner +from .base import (ExecutorException, + Protocol, + RefTestExecutor, + RefTestImplementation, + TestExecutor, + TestharnessExecutor, + testharness_result_converter, + reftest_result_converter, + strip_server, + WdspecExecutor) +from ..testrunner import Stop +from ..webdriver_server import GeckoDriverServer + +# Extra timeout to use after internal test timeout at which the harness +# should force a timeout +extra_timeout = 5 # seconds + + +def do_delayed_imports(): + global errors, marionette, webdriver + + # Marionette client used to be called marionette, recently it changed + # to marionette_driver for unfathomable reasons + try: + import marionette + from marionette import errors + except ImportError: + from marionette_driver import marionette, errors + + import webdriver + + +class MarionetteProtocol(Protocol): + def __init__(self, executor, browser): + do_delayed_imports() + + Protocol.__init__(self, executor, browser) + self.marionette = None + self.marionette_port = browser.marionette_port + self.timeout = None + self.runner_handle = None + + def setup(self, runner): + """Connect to browser via Marionette.""" + Protocol.setup(self, runner) + + self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port) + self.marionette = marionette.Marionette(host='localhost', + port=self.marionette_port, + socket_timeout=None) + + # XXX Move this timeout somewhere + self.logger.debug("Waiting for Marionette connection") + while True: + success = self.marionette.wait_for_port(60) + #When running in a debugger wait indefinitely for firefox to start + if success or self.executor.debug_info is None: + break + + session_started = False + if success: + try: + self.logger.debug("Starting Marionette session") + self.marionette.start_session() + except Exception as e: + self.logger.warning("Starting marionette session failed: %s" % e) + else: + self.logger.debug("Marionette session started") + session_started = True + + if not success or not session_started: + self.logger.warning("Failed to connect to Marionette") + self.executor.runner.send_message("init_failed") + else: + try: + self.after_connect() + except Exception: + self.logger.warning("Post-connection steps failed") + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + try: + self.marionette.delete_session() + except Exception: + # This is typically because the session never started + pass + del self.marionette + + @property + def is_alive(self): + """Check if the Marionette connection is still active.""" + try: + self.marionette.current_window_handle + except Exception: + return False + return True + + def after_connect(self): + self.load_runner("http") + + def set_timeout(self, timeout): + """Set the Marionette script timeout. + + :param timeout: Script timeout in seconds + + """ + self.marionette.timeout.script = timeout + self.timeout = timeout + + def load_runner(self, protocol): + # Check if we previously had a test window open, and if we did make sure it's closed + self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}") + url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html") + self.logger.debug("Loading %s" % url) + self.runner_handle = self.marionette.current_window_handle + try: + self.marionette.navigate(url) + except Exception as e: + self.logger.critical( + "Loading initial page %s failed. Ensure that the " + "there are no other programs bound to this port and " + "that your firewall rules or network setup does not " + "prevent access.\e%s" % (url, traceback.format_exc(e))) + self.marionette.execute_script( + "document.title = '%s'" % threading.current_thread().name.replace("'", '"')) + + def close_old_windows(self, protocol): + handles = self.marionette.window_handles + runner_handle = None + try: + handles.remove(self.runner_handle) + runner_handle = self.runner_handle + except ValueError: + # The runner window probably changed id but we can restore it + # This isn't supposed to happen, but marionette ids are not yet stable + # We assume that the first handle returned corresponds to the runner, + # but it hopefully doesn't matter too much if that assumption is + # wrong since we reload the runner in that tab anyway. + runner_handle = handles.pop(0) + + for handle in handles: + self.marionette.switch_to_window(handle) + self.marionette.close() + + self.marionette.switch_to_window(runner_handle) + if runner_handle != self.runner_handle: + self.load_runner(protocol) + + def wait(self): + socket_timeout = self.marionette.client.sock.gettimeout() + if socket_timeout: + self.marionette.timeout.script = socket_timeout / 2 + + while True: + try: + self.marionette.execute_async_script("") + except errors.ScriptTimeoutException: + self.logger.debug("Script timed out") + pass + except (socket.timeout, IOError): + self.logger.debug("Socket closed") + break + except Exception as e: + self.logger.error(traceback.format_exc(e)) + break + + def on_environment_change(self, old_environment, new_environment): + #Unset all the old prefs + for name in old_environment.get("prefs", {}).iterkeys(): + value = self.executor.original_pref_values[name] + if value is None: + self.clear_user_pref(name) + else: + self.set_pref(name, value) + + for name, value in new_environment.get("prefs", {}).iteritems(): + self.executor.original_pref_values[name] = self.get_pref(name) + self.set_pref(name, value) + + def set_pref(self, name, value): + if value.lower() not in ("true", "false"): + try: + int(value) + except ValueError: + value = "'%s'" % value + else: + value = value.lower() + + self.logger.info("Setting pref %s (%s)" % (name, value)) + + script = """ + let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefBranch); + let pref = '%s'; + let type = prefInterface.getPrefType(pref); + let value = %s; + switch(type) { + case prefInterface.PREF_STRING: + prefInterface.setCharPref(pref, value); + break; + case prefInterface.PREF_BOOL: + prefInterface.setBoolPref(pref, value); + break; + case prefInterface.PREF_INT: + prefInterface.setIntPref(pref, value); + break; + } + """ % (name, value) + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) + + def clear_user_pref(self, name): + self.logger.info("Clearing pref %s" % (name)) + script = """ + let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefBranch); + let pref = '%s'; + prefInterface.clearUserPref(pref); + """ % name + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) + + def get_pref(self, name): + script = """ + let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefBranch); + let pref = '%s'; + let type = prefInterface.getPrefType(pref); + switch(type) { + case prefInterface.PREF_STRING: + return prefInterface.getCharPref(pref); + case prefInterface.PREF_BOOL: + return prefInterface.getBoolPref(pref); + case prefInterface.PREF_INT: + return prefInterface.getIntPref(pref); + case prefInterface.PREF_INVALID: + return null; + } + """ % name + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) + + +class RemoteMarionetteProtocol(Protocol): + def __init__(self, executor, browser): + do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.session = None + self.webdriver_binary = executor.webdriver_binary + self.marionette_port = browser.marionette_port + self.server = None + + def setup(self, runner): + """Connect to browser via the Marionette HTTP server.""" + try: + self.server = GeckoDriverServer( + self.logger, self.marionette_port, binary=self.webdriver_binary) + self.server.start(block=False) + self.logger.info( + "WebDriver HTTP server listening at %s" % self.server.url) + + self.logger.info( + "Establishing new WebDriver session with %s" % self.server.url) + self.session = webdriver.Session( + self.server.host, self.server.port, self.server.base_path) + except Exception: + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + try: + if self.session.session_id is not None: + self.session.end() + except Exception: + pass + if self.server is not None and self.server.is_alive: + self.server.stop() + + @property + def is_alive(self): + """Test that the Marionette connection is still alive. + + Because the remote communication happens over HTTP we need to + make an explicit request to the remote. It is allowed for + WebDriver spec tests to not have a WebDriver session, since this + may be what is tested. + + An HTTP request to an invalid path that results in a 404 is + proof enough to us that the server is alive and kicking. + """ + conn = httplib.HTTPConnection(self.server.host, self.server.port) + conn.request("HEAD", self.server.base_path + "invalid") + res = conn.getresponse() + return res.status == 404 + + +class ExecuteAsyncScriptRun(object): + def __init__(self, logger, func, protocol, url, timeout): + self.logger = logger + self.result = (None, None) + self.protocol = protocol + self.marionette = protocol.marionette + self.func = func + self.url = url + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + timeout = self.timeout + + try: + if timeout is not None: + if timeout + extra_timeout != self.protocol.timeout: + self.protocol.set_timeout(timeout + extra_timeout) + else: + # We just want it to never time out, really, but marionette doesn't + # make that possible. It also seems to time out immediately if the + # timeout is set too high. This works at least. + self.protocol.set_timeout(2**28 - 1) + except IOError: + self.logger.error("Lost marionette connection before starting test") + return Stop + + executor = threading.Thread(target = self._run) + executor.start() + + if timeout is not None: + wait_timeout = timeout + 2 * extra_timeout + else: + wait_timeout = None + + flag = self.result_flag.wait(wait_timeout) + if self.result[1] is None: + self.logger.debug("Timed out waiting for a result") + self.result = False, ("EXTERNAL-TIMEOUT", None) + return self.result + + def _run(self): + try: + self.result = True, self.func(self.marionette, self.url, self.timeout) + except errors.ScriptTimeoutException: + self.logger.debug("Got a marionette timeout") + self.result = False, ("EXTERNAL-TIMEOUT", None) + except (socket.timeout, IOError): + # This can happen on a crash + # Also, should check after the test if the firefox process is still running + # and otherwise ignore any other result and set it to crash + self.result = False, ("CRASH", None) + except Exception as e: + message = getattr(e, "message", "") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", e) + + finally: + self.result_flag.set() + + +class MarionetteTestharnessExecutor(TestharnessExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + close_after_done=True, debug_info=None, **kwargs): + """Marionette-based executor for testharness.js tests""" + TestharnessExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + + self.protocol = MarionetteProtocol(self, browser) + self.script = open(os.path.join(here, "testharness_marionette.js")).read() + self.close_after_done = close_after_done + self.window_id = str(uuid.uuid4()) + + self.original_pref_values = {} + + if marionette is None: + do_delayed_imports() + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + self.protocol.on_environment_change(self.last_environment, new_environment) + + if new_environment["protocol"] != self.last_environment["protocol"]: + self.protocol.load_runner(new_environment["protocol"]) + + def do_test(self, test): + timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None + else None) + + success, data = ExecuteAsyncScriptRun(self.logger, + self.do_testharness, + self.protocol, + self.test_url(test), + timeout).run() + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_testharness(self, marionette, url, timeout): + if self.close_after_done: + marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}") + self.protocol.close_old_windows(self.protocol) + + if timeout is not None: + timeout_ms = str(timeout * 1000) + else: + timeout_ms = "null" + + script = self.script % {"abs_url": url, + "url": strip_server(url), + "window_id": self.window_id, + "timeout_multiplier": self.timeout_multiplier, + "timeout": timeout_ms, + "explicit_timeout": timeout is None} + + rv = marionette.execute_async_script(script, new_sandbox=False) + return rv + + +class MarionetteRefTestExecutor(RefTestExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + screenshot_cache=None, close_after_done=True, + debug_info=None, **kwargs): + + """Marionette-based executor for reftests""" + RefTestExecutor.__init__(self, + browser, + server_config, + screenshot_cache=screenshot_cache, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = MarionetteProtocol(self, browser) + self.implementation = RefTestImplementation(self) + self.close_after_done = close_after_done + self.has_window = False + self.original_pref_values = {} + + with open(os.path.join(here, "reftest.js")) as f: + self.script = f.read() + with open(os.path.join(here, "reftest-wait.js")) as f: + self.wait_script = f.read() + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + self.protocol.on_environment_change(self.last_environment, new_environment) + + def do_test(self, test): + if self.close_after_done and self.has_window: + self.protocol.marionette.close() + self.protocol.marionette.switch_to_window( + self.protocol.marionette.window_handles[-1]) + self.has_window = False + + if not self.has_window: + self.protocol.marionette.execute_script(self.script) + self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1]) + self.has_window = True + + result = self.implementation.run_test(test) + return self.convert_result(test, result) + + def screenshot(self, test, viewport_size, dpi): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + assert dpi is None + + timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None + + test_url = self.test_url(test) + + return ExecuteAsyncScriptRun(self.logger, + self._screenshot, + self.protocol, + test_url, + timeout).run() + + def _screenshot(self, marionette, url, timeout): + marionette.navigate(url) + + marionette.execute_async_script(self.wait_script) + + screenshot = marionette.screenshot() + # strip off the data:img/png, part of the url + if screenshot.startswith("data:image/png;base64,"): + screenshot = screenshot.split(",", 1)[1] + + return screenshot + + +class WdspecRun(object): + def __init__(self, func, session, path, timeout): + self.func = func + self.result = (None, None) + self.session = session + self.path = path + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + """Runs function in a thread and interrupts it if it exceeds the + given timeout. Returns (True, (Result, [SubtestResult ...])) in + case of success, or (False, (status, extra information)) in the + event of failure. + """ + + executor = threading.Thread(target=self._run) + executor.start() + + flag = self.result_flag.wait(self.timeout) + if self.result[1] is None: + self.result = False, ("EXTERNAL-TIMEOUT", None) + + return self.result + + def _run(self): + try: + self.result = True, self.func(self.session, self.path, self.timeout) + except (socket.timeout, IOError): + self.result = False, ("CRASH", None) + except Exception as e: + message = getattr(e, "message") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", message) + finally: + self.result_flag.set() + + +class MarionetteWdspecExecutor(WdspecExecutor): + def __init__(self, browser, server_config, webdriver_binary, + timeout_multiplier=1, close_after_done=True, debug_info=None): + WdspecExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.webdriver_binary = webdriver_binary + self.protocol = RemoteMarionetteProtocol(self, browser) + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + pass + + def do_test(self, test): + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + success, data = WdspecRun(self.do_wdspec, + self.protocol.session, + test.abs_path, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_wdspec(self, session, path, timeout): + harness_result = ("OK", None) + subtest_results = pytestrunner.run( + path, session, self.server_url, timeout=timeout) + return (harness_result, subtest_results) diff --git a/testing/web-platform/harness/wptrunner/executors/executorselenium.py b/testing/web-platform/harness/wptrunner/executors/executorselenium.py new file mode 100644 index 000000000..b8b209a77 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py @@ -0,0 +1,271 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import socket +import sys +import threading +import time +import traceback +import urlparse +import uuid + +from .base import (ExecutorException, + Protocol, + RefTestExecutor, + RefTestImplementation, + TestExecutor, + TestharnessExecutor, + testharness_result_converter, + reftest_result_converter, + strip_server) +from ..testrunner import Stop + + +here = os.path.join(os.path.split(__file__)[0]) + +webdriver = None +exceptions = None + +extra_timeout = 5 + +def do_delayed_imports(): + global webdriver + global exceptions + from selenium import webdriver + from selenium.common import exceptions + + +class SeleniumProtocol(Protocol): + def __init__(self, executor, browser, capabilities, **kwargs): + do_delayed_imports() + + Protocol.__init__(self, executor, browser) + self.capabilities = capabilities + self.url = browser.webdriver_url + self.webdriver = None + + def setup(self, runner): + """Connect to browser via Selenium's WebDriver implementation.""" + self.runner = runner + self.logger.debug("Connecting to Selenium on URL: %s" % self.url) + + session_started = False + try: + self.webdriver = webdriver.Remote( + self.url, desired_capabilities=self.capabilities) + except: + self.logger.warning( + "Connecting to Selenium failed:\n%s" % traceback.format_exc()) + else: + self.logger.debug("Selenium session started") + session_started = True + + if not session_started: + self.logger.warning("Failed to connect to Selenium") + self.executor.runner.send_message("init_failed") + else: + try: + self.after_connect() + except: + print >> sys.stderr, traceback.format_exc() + self.logger.warning( + "Failed to connect to navigate initial page") + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + self.logger.debug("Hanging up on Selenium session") + try: + self.webdriver.quit() + except: + pass + del self.webdriver + + def is_alive(self): + try: + # Get a simple property over the connection + self.webdriver.current_window_handle + # TODO what exception? + except (socket.timeout, exceptions.ErrorInResponseException): + return False + return True + + def after_connect(self): + self.load_runner("http") + + def load_runner(self, protocol): + url = urlparse.urljoin(self.executor.server_url(protocol), + "/testharness_runner.html") + self.logger.debug("Loading %s" % url) + self.webdriver.get(url) + self.webdriver.execute_script("document.title = '%s'" % + threading.current_thread().name.replace("'", '"')) + + def wait(self): + while True: + try: + self.webdriver.execute_async_script(""); + except exceptions.TimeoutException: + pass + except (socket.timeout, exceptions.NoSuchWindowException, + exceptions.ErrorInResponseException, IOError): + break + except Exception as e: + self.logger.error(traceback.format_exc(e)) + break + + +class SeleniumRun(object): + def __init__(self, func, webdriver, url, timeout): + self.func = func + self.result = None + self.webdriver = webdriver + self.url = url + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + timeout = self.timeout + + try: + self.webdriver.set_script_timeout((timeout + extra_timeout) * 1000) + except exceptions.ErrorInResponseException: + self.logger.error("Lost WebDriver connection") + return Stop + + executor = threading.Thread(target=self._run) + executor.start() + + flag = self.result_flag.wait(timeout + 2 * extra_timeout) + if self.result is None: + assert not flag + self.result = False, ("EXTERNAL-TIMEOUT", None) + + return self.result + + def _run(self): + try: + self.result = True, self.func(self.webdriver, self.url, self.timeout) + except exceptions.TimeoutException: + self.result = False, ("EXTERNAL-TIMEOUT", None) + except (socket.timeout, exceptions.ErrorInResponseException): + self.result = False, ("CRASH", None) + except Exception as e: + message = getattr(e, "message", "") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", e) + finally: + self.result_flag.set() + + +class SeleniumTestharnessExecutor(TestharnessExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + close_after_done=True, capabilities=None, debug_info=None): + """Selenium-based executor for testharness.js tests""" + TestharnessExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = SeleniumProtocol(self, browser, capabilities) + with open(os.path.join(here, "testharness_webdriver.js")) as f: + self.script = f.read() + self.close_after_done = close_after_done + self.window_id = str(uuid.uuid4()) + + def is_alive(self): + return self.protocol.is_alive() + + def on_protocol_change(self, new_protocol): + self.protocol.load_runner(new_protocol) + + def do_test(self, test): + url = self.test_url(test) + + success, data = SeleniumRun(self.do_testharness, + self.protocol.webdriver, + url, + test.timeout * self.timeout_multiplier).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_testharness(self, webdriver, url, timeout): + return webdriver.execute_async_script( + self.script % {"abs_url": url, + "url": strip_server(url), + "window_id": self.window_id, + "timeout_multiplier": self.timeout_multiplier, + "timeout": timeout * 1000}) + +class SeleniumRefTestExecutor(RefTestExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + screenshot_cache=None, close_after_done=True, + debug_info=None, capabilities=None): + """Selenium WebDriver-based executor for reftests""" + RefTestExecutor.__init__(self, + browser, + server_config, + screenshot_cache=screenshot_cache, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = SeleniumProtocol(self, browser, + capabilities=capabilities) + self.implementation = RefTestImplementation(self) + self.close_after_done = close_after_done + self.has_window = False + + with open(os.path.join(here, "reftest.js")) as f: + self.script = f.read() + with open(os.path.join(here, "reftest-wait_webdriver.js")) as f: + self.wait_script = f.read() + + def is_alive(self): + return self.protocol.is_alive() + + def do_test(self, test): + self.logger.info("Test requires OS-level window focus") + + if self.close_after_done and self.has_window: + self.protocol.webdriver.close() + self.protocol.webdriver.switch_to_window( + self.protocol.webdriver.window_handles[-1]) + self.has_window = False + + if not self.has_window: + self.protocol.webdriver.execute_script(self.script) + self.protocol.webdriver.switch_to_window( + self.protocol.webdriver.window_handles[-1]) + self.has_window = True + + result = self.implementation.run_test(test) + + return self.convert_result(test, result) + + def screenshot(self, test, viewport_size, dpi): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + assert dpi is None + + return SeleniumRun(self._screenshot, + self.protocol.webdriver, + self.test_url(test), + test.timeout).run() + + def _screenshot(self, webdriver, url, timeout): + webdriver.get(url) + + webdriver.execute_async_script(self.wait_script) + + screenshot = webdriver.get_screenshot_as_base64() + + # strip off the data:img/png, part of the url + if screenshot.startswith("data:image/png;base64,"): + screenshot = screenshot.split(",", 1)[1] + + return screenshot diff --git a/testing/web-platform/harness/wptrunner/executors/executorservo.py b/testing/web-platform/harness/wptrunner/executors/executorservo.py new file mode 100644 index 000000000..068061b95 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py @@ -0,0 +1,275 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import base64 +import hashlib +import json +import os +import subprocess +import tempfile +import threading +import urlparse +import uuid +from collections import defaultdict + +from mozprocess import ProcessHandler + +from .base import (ExecutorException, + Protocol, + RefTestImplementation, + testharness_result_converter, + reftest_result_converter) +from .process import ProcessTestExecutor +from ..browsers.base import browser_command +render_arg = None + + +def do_delayed_imports(): + global render_arg + from ..browsers.servo import render_arg + +hosts_text = """127.0.0.1 web-platform.test +127.0.0.1 www.web-platform.test +127.0.0.1 www1.web-platform.test +127.0.0.1 www2.web-platform.test +127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test +127.0.0.1 xn--lve-6lad.web-platform.test +""" + +def make_hosts_file(): + hosts_fd, hosts_path = tempfile.mkstemp() + with os.fdopen(hosts_fd, "w") as f: + f.write(hosts_text) + return hosts_path + + +class ServoTestharnessExecutor(ProcessTestExecutor): + convert_result = testharness_result_converter + + def __init__(self, browser, server_config, timeout_multiplier=1, debug_info=None, + pause_after_test=False): + do_delayed_imports() + ProcessTestExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.pause_after_test = pause_after_test + self.result_data = None + self.result_flag = None + self.protocol = Protocol(self, browser) + self.hosts_path = make_hosts_file() + + def teardown(self): + try: + os.unlink(self.hosts_path) + except OSError: + pass + ProcessTestExecutor.teardown(self) + + def do_test(self, test): + self.result_data = None + self.result_flag = threading.Event() + + args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner", + "-Z", "replace-surrogates", "-z", self.test_url(test)] + for stylesheet in self.browser.user_stylesheets: + args += ["--user-stylesheet", stylesheet] + for pref, value in test.environment.get('prefs', {}).iteritems(): + args += ["--pref", "%s=%s" % (pref, value)] + args += self.browser.binary_args + debug_args, command = browser_command(self.binary, args, self.debug_info) + + self.command = command + + if self.pause_after_test: + self.command.remove("-z") + + self.command = debug_args + self.command + + env = os.environ.copy() + env["HOST_FILE"] = self.hosts_path + env["RUST_BACKTRACE"] = "1" + + + if not self.interactive: + self.proc = ProcessHandler(self.command, + processOutputLine=[self.on_output], + onFinish=self.on_finish, + env=env, + storeOutput=False) + self.proc.run() + else: + self.proc = subprocess.Popen(self.command, env=env) + + try: + timeout = test.timeout * self.timeout_multiplier + + # Now wait to get the output we expect, or until we reach the timeout + if not self.interactive and not self.pause_after_test: + wait_timeout = timeout + 5 + self.result_flag.wait(wait_timeout) + else: + wait_timeout = None + self.proc.wait() + + proc_is_running = True + + if self.result_flag.is_set(): + if self.result_data is not None: + result = self.convert_result(test, self.result_data) + else: + self.proc.wait() + result = (test.result_cls("CRASH", None), []) + proc_is_running = False + else: + result = (test.result_cls("TIMEOUT", None), []) + + + if proc_is_running: + if self.pause_after_test: + self.logger.info("Pausing until the browser exits") + self.proc.wait() + else: + self.proc.kill() + except KeyboardInterrupt: + self.proc.kill() + raise + + return result + + def on_output(self, line): + prefix = "ALERT: RESULT: " + line = line.decode("utf8", "replace") + if line.startswith(prefix): + self.result_data = json.loads(line[len(prefix):]) + self.result_flag.set() + else: + if self.interactive: + print line + else: + self.logger.process_output(self.proc.pid, + line, + " ".join(self.command)) + + def on_finish(self): + self.result_flag.set() + + +class TempFilename(object): + def __init__(self, directory): + self.directory = directory + self.path = None + + def __enter__(self): + self.path = os.path.join(self.directory, str(uuid.uuid4())) + return self.path + + def __exit__(self, *args, **kwargs): + try: + os.unlink(self.path) + except OSError: + pass + + +class ServoRefTestExecutor(ProcessTestExecutor): + convert_result = reftest_result_converter + + def __init__(self, browser, server_config, binary=None, timeout_multiplier=1, + screenshot_cache=None, debug_info=None, pause_after_test=False): + do_delayed_imports() + ProcessTestExecutor.__init__(self, + browser, + server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + + self.protocol = Protocol(self, browser) + self.screenshot_cache = screenshot_cache + self.implementation = RefTestImplementation(self) + self.tempdir = tempfile.mkdtemp() + self.hosts_path = make_hosts_file() + + def teardown(self): + try: + os.unlink(self.hosts_path) + except OSError: + pass + os.rmdir(self.tempdir) + ProcessTestExecutor.teardown(self) + + def screenshot(self, test, viewport_size, dpi): + full_url = self.test_url(test) + + with TempFilename(self.tempdir) as output_path: + debug_args, command = browser_command( + self.binary, + [render_arg(self.browser.render_backend), "--hard-fail", "--exit", + "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates", + "--output=%s" % output_path, full_url], + self.debug_info) + + for stylesheet in self.browser.user_stylesheets: + command += ["--user-stylesheet", stylesheet] + + for pref in test.environment.get('prefs', {}): + command += ["--pref", pref] + + if viewport_size: + command += ["--resolution", viewport_size] + + if dpi: + command += ["--device-pixel-ratio", dpi] + + self.command = debug_args + command + + env = os.environ.copy() + env["HOST_FILE"] = self.hosts_path + env["RUST_BACKTRACE"] = "1" + + if not self.interactive: + self.proc = ProcessHandler(self.command, + processOutputLine=[self.on_output], + env=env) + + + try: + self.proc.run() + timeout = test.timeout * self.timeout_multiplier + 5 + rv = self.proc.wait(timeout=timeout) + except KeyboardInterrupt: + self.proc.kill() + raise + else: + self.proc = subprocess.Popen(self.command, + env=env) + try: + rv = self.proc.wait() + except KeyboardInterrupt: + self.proc.kill() + raise + + if rv is None: + self.proc.kill() + return False, ("EXTERNAL-TIMEOUT", None) + + if rv != 0 or not os.path.exists(output_path): + return False, ("CRASH", None) + + with open(output_path) as f: + # Might need to strip variable headers or something here + data = f.read() + return True, base64.b64encode(data) + + def do_test(self, test): + result = self.implementation.run_test(test) + + return self.convert_result(test, result) + + def on_output(self, line): + line = line.decode("utf8", "replace") + if self.interactive: + print line + else: + self.logger.process_output(self.proc.pid, + line, + " ".join(self.command)) diff --git a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py new file mode 100644 index 000000000..fceeb58fa --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py @@ -0,0 +1,262 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import socket +import threading +import time +import traceback + +from .base import (Protocol, + RefTestExecutor, + RefTestImplementation, + TestharnessExecutor, + strip_server) +from .. import webdriver +from ..testrunner import Stop + +webdriver = None + +here = os.path.join(os.path.split(__file__)[0]) + +extra_timeout = 5 + + +def do_delayed_imports(): + global webdriver + import webdriver + + +class ServoWebDriverProtocol(Protocol): + def __init__(self, executor, browser, capabilities, **kwargs): + do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.capabilities = capabilities + self.host = browser.webdriver_host + self.port = browser.webdriver_port + self.session = None + + def setup(self, runner): + """Connect to browser via WebDriver.""" + self.runner = runner + + url = "http://%s:%d" % (self.host, self.port) + session_started = False + try: + self.session = webdriver.Session(self.host, self.port, + extension=webdriver.servo.ServoCommandExtensions) + self.session.start() + except: + self.logger.warning( + "Connecting with WebDriver failed:\n%s" % traceback.format_exc()) + else: + self.logger.debug("session started") + session_started = True + + if not session_started: + self.logger.warning("Failed to connect via WebDriver") + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + self.logger.debug("Hanging up on WebDriver session") + try: + self.session.end() + except: + pass + + def is_alive(self): + try: + # Get a simple property over the connection + self.session.window_handle + # TODO what exception? + except Exception: + return False + return True + + def after_connect(self): + pass + + def wait(self): + while True: + try: + self.session.execute_async_script("") + except webdriver.TimeoutException: + pass + except (socket.timeout, IOError): + break + except Exception as e: + self.logger.error(traceback.format_exc(e)) + break + + def on_environment_change(self, old_environment, new_environment): + #Unset all the old prefs + self.session.extension.reset_prefs(*old_environment.get("prefs", {}).keys()) + self.session.extension.set_prefs(new_environment.get("prefs", {})) + + +class ServoWebDriverRun(object): + def __init__(self, func, session, url, timeout, current_timeout=None): + self.func = func + self.result = None + self.session = session + self.url = url + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + executor = threading.Thread(target=self._run) + executor.start() + + flag = self.result_flag.wait(self.timeout + extra_timeout) + if self.result is None: + assert not flag + self.result = False, ("EXTERNAL-TIMEOUT", None) + + return self.result + + def _run(self): + try: + self.result = True, self.func(self.session, self.url, self.timeout) + except webdriver.TimeoutException: + self.result = False, ("EXTERNAL-TIMEOUT", None) + except (socket.timeout, IOError): + self.result = False, ("CRASH", None) + except Exception as e: + message = getattr(e, "message", "") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", e) + finally: + self.result_flag.set() + + +def timeout_func(timeout): + if timeout: + t0 = time.time() + return lambda: time.time() - t0 > timeout + extra_timeout + else: + return lambda: False + + +class ServoWebDriverTestharnessExecutor(TestharnessExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + close_after_done=True, capabilities=None, debug_info=None): + TestharnessExecutor.__init__(self, browser, server_config, timeout_multiplier=1, + debug_info=None) + self.protocol = ServoWebDriverProtocol(self, browser, capabilities=capabilities) + with open(os.path.join(here, "testharness_servodriver.js")) as f: + self.script = f.read() + self.timeout = None + + def on_protocol_change(self, new_protocol): + pass + + def is_alive(self): + return self.protocol.is_alive() + + def do_test(self, test): + url = self.test_url(test) + + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + if timeout != self.timeout: + try: + self.protocol.session.timeouts.script = timeout + self.timeout = timeout + except IOError: + self.logger.error("Lost webdriver connection") + return Stop + + success, data = ServoWebDriverRun(self.do_testharness, + self.protocol.session, + url, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_testharness(self, session, url, timeout): + session.url = url + result = json.loads( + session.execute_async_script( + self.script % {"abs_url": url, + "url": strip_server(url), + "timeout_multiplier": self.timeout_multiplier, + "timeout": timeout * 1000})) + # Prevent leaking every page in history until Servo develops a more sane + # page cache + session.back() + return result + + +class TimeoutError(Exception): + pass + + +class ServoWebDriverRefTestExecutor(RefTestExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + screenshot_cache=None, capabilities=None, debug_info=None): + """Selenium WebDriver-based executor for reftests""" + RefTestExecutor.__init__(self, + browser, + server_config, + screenshot_cache=screenshot_cache, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = ServoWebDriverProtocol(self, browser, + capabilities=capabilities) + self.implementation = RefTestImplementation(self) + self.timeout = None + with open(os.path.join(here, "reftest-wait_servodriver.js")) as f: + self.wait_script = f.read() + + def is_alive(self): + return self.protocol.is_alive() + + def do_test(self, test): + try: + result = self.implementation.run_test(test) + return self.convert_result(test, result) + except IOError: + return test.result_cls("CRASH", None), [] + except TimeoutError: + return test.result_cls("TIMEOUT", None), [] + except Exception as e: + message = getattr(e, "message", "") + if message: + message += "\n" + message += traceback.format_exc(e) + return test.result_cls("ERROR", message), [] + + def screenshot(self, test, viewport_size, dpi): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + assert dpi is None + + timeout = (test.timeout * self.timeout_multiplier + extra_timeout + if self.debug_info is None else None) + + if self.timeout != timeout: + try: + self.protocol.session.timeouts.script = timeout + self.timeout = timeout + except IOError: + self.logger.error("Lost webdriver connection") + return Stop + + return ServoWebDriverRun(self._screenshot, + self.protocol.session, + self.test_url(test), + timeout).run() + + def _screenshot(self, session, url, timeout): + session.url = url + session.execute_async_script(self.wait_script) + return session.screenshot() diff --git a/testing/web-platform/harness/wptrunner/executors/process.py b/testing/web-platform/harness/wptrunner/executors/process.py new file mode 100644 index 000000000..45f33ab2c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/process.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .base import TestExecutor + + +class ProcessTestExecutor(TestExecutor): + def __init__(self, *args, **kwargs): + TestExecutor.__init__(self, *args, **kwargs) + self.binary = self.browser.binary + self.interactive = (False if self.debug_info is None + else self.debug_info.interactive) + + def setup(self, runner): + self.runner = runner + self.runner.send_message("init_succeeded") + return True + + def is_alive(self): + return True + + def do_test(self, test): + raise NotImplementedError diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py new file mode 100644 index 000000000..de3a34a79 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from . import fixtures +from .runner import run diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py new file mode 100644 index 000000000..1b4e8d43d --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +import urlparse + + +"""pytest fixtures for use in Python-based WPT tests. + +The purpose of test fixtures is to provide a fixed baseline upon which +tests can reliably and repeatedly execute. +""" + + +class Session(object): + """Fixture to allow access to wptrunner's existing WebDriver session + in tests. + + The session is not created by default to enable testing of session + creation. However, a module-scoped session will be implicitly created + at the first call to a WebDriver command. This means methods such as + `session.send_command` and `session.session_id` are possible to use + without having a session. + + To illustrate implicit session creation:: + + def test_session_scope(session): + # at this point there is no session + assert session.session_id is None + + # window_id is a WebDriver command, + # and implicitly creates the session for us + assert session.window_id is not None + + # we now have a session + assert session.session_id is not None + + You can also access the session in custom fixtures defined in the + tests, such as a setup function:: + + @pytest.fixture(scope="function") + def setup(request, session): + session.url = "https://example.org" + + def test_something(setup, session): + assert session.url == "https://example.org" + + The session is closed when the test module goes out of scope by an + implicit call to `session.end`. + """ + + def __init__(self, client): + self.client = client + + @pytest.fixture(scope="module") + def session(self, request): + request.addfinalizer(self.client.end) + return self.client + +class Server(object): + """Fixture to allow access to wptrunner's base server url. + + :param url_getter: Function to get server url from test environment, given + a protocol. + """ + def __init__(self, url_getter): + self.server_url = url_getter + + def where_is(self, uri, protocol="http"): + return urlparse.urljoin(self.server_url(protocol), uri) + + @pytest.fixture + def server(self, request): + return self diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py new file mode 100644 index 000000000..28b8f609c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Provides interface to deal with pytest. + +Usage:: + + session = webdriver.client.Session("127.0.0.1", "4444", "/") + harness_result = ("OK", None) + subtest_results = pytestrunner.run("/path/to/test", session.url) + return (harness_result, subtest_results) +""" + +import errno +import shutil +import tempfile + +from . import fixtures + + +pytest = None + + +def do_delayed_imports(): + global pytest + import pytest + + +def run(path, session, url_getter, timeout=0): + """Run Python test at ``path`` in pytest. The provided ``session`` + is exposed as a fixture available in the scope of the test functions. + + :param path: Path to the test file. + :param session: WebDriver session to expose. + :param url_getter: Function to get server url from test environment, given + a protocol. + :param timeout: Duration before interrupting potentially hanging + tests. If 0, there is no timeout. + + :returns: List of subtest results, which are tuples of (test id, + status, message, stacktrace). + """ + + if pytest is None: + do_delayed_imports() + + recorder = SubtestResultRecorder() + plugins = [recorder, + fixtures.Session(session), + fixtures.Server(url_getter)] + + # TODO(ato): Deal with timeouts + + with TemporaryDirectory() as cache: + pytest.main(["--strict", # turn warnings into errors + "--verbose", # show each individual subtest + "--capture", "no", # enable stdout/stderr from tests + "--basetemp", cache, # temporary directory + path], + plugins=plugins) + + return recorder.results + + +class SubtestResultRecorder(object): + def __init__(self): + self.results = [] + + def pytest_runtest_logreport(self, report): + if report.passed and report.when == "call": + self.record_pass(report) + elif report.failed: + if report.when != "call": + self.record_error(report) + else: + self.record_fail(report) + elif report.skipped: + self.record_skip(report) + + def record_pass(self, report): + self.record(report.nodeid, "PASS") + + def record_fail(self, report): + self.record(report.nodeid, "FAIL", stack=report.longrepr) + + def record_error(self, report): + # error in setup/teardown + if report.when != "call": + message = "%s error" % report.when + self.record(report.nodeid, "ERROR", message, report.longrepr) + + def record_skip(self, report): + self.record(report.nodeid, "ERROR", + "In-test skip decorators are disallowed, " + "please use WPT metadata to ignore tests.") + + def record(self, test, status, message=None, stack=None): + if stack is not None: + stack = str(stack) + new_result = (test, status, message, stack) + self.results.append(new_result) + + +class TemporaryDirectory(object): + def __enter__(self): + self.path = tempfile.mkdtemp(prefix="pytest-") + return self.path + + def __exit__(self, *args): + try: + shutil.rmtree(self.path) + except OSError as e: + # no such file or directory + if e.errno != errno.ENOENT: + raise diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait.js new file mode 100644 index 000000000..a2bc66565 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait.js @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test(x) { + log("classList: " + root.classList); + if (!root.classList.contains("reftest-wait")) { + observer.disconnect(); + marionetteScriptFinished(); + } +} + +var root = document.documentElement; +var observer = new MutationObserver(test); + +observer.observe(root, {attributes: true}); + +if (document.readyState != "complete") { + onload = test +} else { + test(); +} diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js new file mode 100644 index 000000000..6040b4336 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js @@ -0,0 +1,20 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +callback = arguments[arguments.length - 1]; + +function check_done() { + if (!document.body.classList.contains('reftest-wait')) { + callback(); + } else { + setTimeout(check_done, 50); + } +} + +if (document.readyState === 'complete') { + check_done(); +} else { + addEventListener("load", check_done); +} diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js new file mode 100644 index 000000000..187f5daac --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var callback = arguments[arguments.length - 1]; + +function test(x) { + if (!root.classList.contains("reftest-wait")) { + observer.disconnect(); + callback() + } +} + +var root = document.documentElement; +var observer = new MutationObserver(test); + +observer.observe(root, {attributes: true}); + +if (document.readyState != "complete") { + onload = test; +} else { + test(); +} diff --git a/testing/web-platform/harness/wptrunner/executors/reftest.js b/testing/web-platform/harness/wptrunner/executors/reftest.js new file mode 100644 index 000000000..3d7a9b6d2 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/reftest.js @@ -0,0 +1,5 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var win = window.open("about:blank", "test", "width=600,height=600"); diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js b/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js new file mode 100644 index 000000000..0d0e4017a --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +window.wrappedJSObject.timeout_multiplier = %(timeout_multiplier)d; +window.wrappedJSObject.explicit_timeout = %(explicit_timeout)d; + +window.wrappedJSObject.addEventListener("message", function listener(event) { + if (event.data.type != "complete") { + return; + } + window.wrappedJSObject.removeEventListener("message", listener); + clearTimeout(timer); + var tests = event.data.tests; + var status = event.data.status; + + var subtest_results = tests.map(function (x) { + return [x.name, x.status, x.message, x.stack] + }); + + marionetteScriptFinished(["%(url)s", + status.status, + status.message, + status.stack, + subtest_results]); +}, false); + +window.wrappedJSObject.win = window.open("%(abs_url)s", "%(window_id)s"); + +var timer = null; +if (%(timeout)s) { + timer = setTimeout(function() { + log("Timeout fired"); + window.wrappedJSObject.win.timeout(); + }, %(timeout)s); +} diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js b/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js new file mode 100644 index 000000000..20aeb34b9 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js @@ -0,0 +1,6 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +window.__wd_results_callback__ = arguments[arguments.length - 1]; +window.__wd_results_timer__ = setTimeout(timeout, %(timeout)s); diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js new file mode 100644 index 000000000..7ca8d5737 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var callback = arguments[arguments.length - 1]; +window.timeout_multiplier = %(timeout_multiplier)d; + +window.addEventListener("message", function(event) { + var tests = event.data[0]; + var status = event.data[1]; + + var subtest_results = tests.map(function(x) { + return [x.name, x.status, x.message, x.stack] + }); + + clearTimeout(timer); + callback(["%(url)s", + status.status, + status.message, + status.stack, + subtest_results]); +}, false); + +window.win = window.open("%(abs_url)s", "%(window_id)s"); + +var timer = setTimeout(function() { + window.win.timeout(); + window.win.close(); +}, %(timeout)s); |