summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/harness/wptrunner/executors
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /testing/web-platform/harness/wptrunner/executors
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'testing/web-platform/harness/wptrunner/executors')
-rw-r--r--testing/web-platform/harness/wptrunner/executors/__init__.py8
-rw-r--r--testing/web-platform/harness/wptrunner/executors/base.py329
-rw-r--r--testing/web-platform/harness/wptrunner/executors/executormarionette.py595
-rw-r--r--testing/web-platform/harness/wptrunner/executors/executorselenium.py271
-rw-r--r--testing/web-platform/harness/wptrunner/executors/executorservo.py275
-rw-r--r--testing/web-platform/harness/wptrunner/executors/executorservodriver.py262
-rw-r--r--testing/web-platform/harness/wptrunner/executors/process.py24
-rw-r--r--testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py6
-rw-r--r--testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py76
-rw-r--r--testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py116
-rw-r--r--testing/web-platform/harness/wptrunner/executors/reftest-wait.js22
-rw-r--r--testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js20
-rw-r--r--testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js23
-rw-r--r--testing/web-platform/harness/wptrunner/executors/reftest.js5
-rw-r--r--testing/web-platform/harness/wptrunner/executors/testharness_marionette.js36
-rw-r--r--testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js6
-rw-r--r--testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js29
17 files changed, 2103 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/executors/__init__.py b/testing/web-platform/harness/wptrunner/executors/__init__.py
new file mode 100644
index 000000000..11e0c4cd7
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/__init__.py
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from base import (executor_kwargs,
+ testharness_result_converter,
+ reftest_result_converter,
+ TestExecutor)
diff --git a/testing/web-platform/harness/wptrunner/executors/base.py b/testing/web-platform/harness/wptrunner/executors/base.py
new file mode 100644
index 000000000..f0ce16658
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/base.py
@@ -0,0 +1,329 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+import json
+import os
+import traceback
+import urlparse
+from abc import ABCMeta, abstractmethod
+
+from ..testrunner import Stop
+
+here = os.path.split(__file__)[0]
+
+
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
+ timeout_multiplier = kwargs["timeout_multiplier"]
+ if timeout_multiplier is None:
+ timeout_multiplier = 1
+
+ executor_kwargs = {"server_config": server_config,
+ "timeout_multiplier": timeout_multiplier,
+ "debug_info": kwargs["debug_info"]}
+
+ if test_type == "reftest":
+ executor_kwargs["screenshot_cache"] = cache_manager.dict()
+
+ return executor_kwargs
+
+
+def strip_server(url):
+ """Remove the scheme and netloc from a url, leaving only the path and any query
+ or fragment.
+
+ url - the url to strip
+
+ e.g. http://example.org:8000/tests?id=1#2 becomes /tests?id=1#2"""
+
+ url_parts = list(urlparse.urlsplit(url))
+ url_parts[0] = ""
+ url_parts[1] = ""
+ return urlparse.urlunsplit(url_parts)
+
+
+class TestharnessResultConverter(object):
+ harness_codes = {0: "OK",
+ 1: "ERROR",
+ 2: "TIMEOUT"}
+
+ test_codes = {0: "PASS",
+ 1: "FAIL",
+ 2: "TIMEOUT",
+ 3: "NOTRUN"}
+
+ def __call__(self, test, result):
+ """Convert a JSON result into a (TestResult, [SubtestResult]) tuple"""
+ result_url, status, message, stack, subtest_results = result
+ assert result_url == test.url, ("Got results from %s, expected %s" %
+ (result_url, test.url))
+ harness_result = test.result_cls(self.harness_codes[status], message)
+ return (harness_result,
+ [test.subtest_result_cls(name, self.test_codes[status], message, stack)
+ for name, status, message, stack in subtest_results])
+
+
+testharness_result_converter = TestharnessResultConverter()
+
+
+def reftest_result_converter(self, test, result):
+ return (test.result_cls(result["status"], result["message"],
+ extra=result.get("extra")), [])
+
+
+def pytest_result_converter(self, test, data):
+ harness_data, subtest_data = data
+
+ if subtest_data is None:
+ subtest_data = []
+
+ harness_result = test.result_cls(*harness_data)
+ subtest_results = [test.subtest_result_cls(*item) for item in subtest_data]
+
+ return (harness_result, subtest_results)
+
+
+class ExecutorException(Exception):
+ def __init__(self, status, message):
+ self.status = status
+ self.message = message
+
+
+class TestExecutor(object):
+ __metaclass__ = ABCMeta
+
+ test_type = None
+ convert_result = None
+
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ debug_info=None):
+ """Abstract Base class for object that actually executes the tests in a
+ specific browser. Typically there will be a different TestExecutor
+ subclass for each test type and method of executing tests.
+
+ :param browser: ExecutorBrowser instance providing properties of the
+ browser that will be tested.
+ :param server_config: Dictionary of wptserve server configuration of the
+ form stored in TestEnvironment.external_config
+ :param timeout_multiplier: Multiplier relative to base timeout to use
+ when setting test timeout.
+ """
+ self.runner = None
+ self.browser = browser
+ self.server_config = server_config
+ self.timeout_multiplier = timeout_multiplier
+ self.debug_info = debug_info
+ self.last_environment = {"protocol": "http",
+ "prefs": {}}
+ self.protocol = None # This must be set in subclasses
+
+ @property
+ def logger(self):
+ """StructuredLogger for this executor"""
+ if self.runner is not None:
+ return self.runner.logger
+
+ def setup(self, runner):
+ """Run steps needed before tests can be started e.g. connecting to
+ browser instance
+
+ :param runner: TestRunner instance that is going to run the tests"""
+ self.runner = runner
+ if self.protocol is not None:
+ self.protocol.setup(runner)
+
+ def teardown(self):
+ """Run cleanup steps after tests have finished"""
+ if self.protocol is not None:
+ self.protocol.teardown()
+
+ def run_test(self, test):
+ """Run a particular test.
+
+ :param test: The test to run"""
+ if test.environment != self.last_environment:
+ self.on_environment_change(test.environment)
+
+ try:
+ result = self.do_test(test)
+ except Exception as e:
+ result = self.result_from_exception(test, e)
+
+ if result is Stop:
+ return result
+
+ # log result of parent test
+ if result[0].status == "ERROR":
+ self.logger.debug(result[0].message)
+
+ self.last_environment = test.environment
+
+ self.runner.send_message("test_ended", test, result)
+
+ def server_url(self, protocol):
+ return "%s://%s:%s" % (protocol,
+ self.server_config["host"],
+ self.server_config["ports"][protocol][0])
+
+ def test_url(self, test):
+ return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url)
+
+ @abstractmethod
+ def do_test(self, test):
+ """Test-type and protocol specific implementation of running a
+ specific test.
+
+ :param test: The test to run."""
+ pass
+
+ def on_environment_change(self, new_environment):
+ pass
+
+ def result_from_exception(self, test, e):
+ if hasattr(e, "status") and e.status in test.result_cls.statuses:
+ status = e.status
+ else:
+ status = "ERROR"
+ message = unicode(getattr(e, "message", ""))
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ return test.result_cls(status, message), []
+
+
+class TestharnessExecutor(TestExecutor):
+ convert_result = testharness_result_converter
+
+
+class RefTestExecutor(TestExecutor):
+ convert_result = reftest_result_converter
+
+ def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
+ debug_info=None):
+ TestExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+
+ self.screenshot_cache = screenshot_cache
+
+
+class RefTestImplementation(object):
+ def __init__(self, executor):
+ self.timeout_multiplier = executor.timeout_multiplier
+ self.executor = executor
+ # Cache of url:(screenshot hash, screenshot). Typically the
+ # screenshot is None, but we set this value if a test fails
+ # and the screenshot was taken from the cache so that we may
+ # retrieve the screenshot from the cache directly in the future
+ self.screenshot_cache = self.executor.screenshot_cache
+ self.message = None
+
+ @property
+ def logger(self):
+ return self.executor.logger
+
+ def get_hash(self, test, viewport_size, dpi):
+ timeout = test.timeout * self.timeout_multiplier
+ key = (test.url, viewport_size, dpi)
+
+ if key not in self.screenshot_cache:
+ success, data = self.executor.screenshot(test, viewport_size, dpi)
+
+ if not success:
+ return False, data
+
+ screenshot = data
+ hash_value = hashlib.sha1(screenshot).hexdigest()
+
+ self.screenshot_cache[key] = (hash_value, None)
+
+ rv = (hash_value, screenshot)
+ else:
+ rv = self.screenshot_cache[key]
+
+ self.message.append("%s %s" % (test.url, rv[0]))
+ return True, rv
+
+ def is_pass(self, lhs_hash, rhs_hash, relation):
+ assert relation in ("==", "!=")
+ self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
+ return ((relation == "==" and lhs_hash == rhs_hash) or
+ (relation == "!=" and lhs_hash != rhs_hash))
+
+ def run_test(self, test):
+ viewport_size = test.viewport_size
+ dpi = test.dpi
+ self.message = []
+
+ # Depth-first search of reference tree, with the goal
+ # of reachings a leaf node with only pass results
+
+ stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
+ while stack:
+ hashes = [None, None]
+ screenshots = [None, None]
+
+ nodes, relation = stack.pop()
+
+ for i, node in enumerate(nodes):
+ success, data = self.get_hash(node, viewport_size, dpi)
+ if success is False:
+ return {"status": data[0], "message": data[1]}
+
+ hashes[i], screenshots[i] = data
+
+ if self.is_pass(hashes[0], hashes[1], relation):
+ if nodes[1].references:
+ stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
+ else:
+ # We passed
+ return {"status":"PASS", "message": None}
+
+ # We failed, so construct a failure message
+
+ for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
+ if screenshot is None:
+ success, screenshot = self.retake_screenshot(node, viewport_size, dpi)
+ if success:
+ screenshots[i] = screenshot
+
+ log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
+ {"url": nodes[1].url, "screenshot": screenshots[1]}]
+
+ return {"status": "FAIL",
+ "message": "\n".join(self.message),
+ "extra": {"reftest_screenshots": log_data}}
+
+ def retake_screenshot(self, node, viewport_size, dpi):
+ success, data = self.executor.screenshot(node, viewport_size, dpi)
+ if not success:
+ return False, data
+
+ key = (node.url, viewport_size, dpi)
+ hash_val, _ = self.screenshot_cache[key]
+ self.screenshot_cache[key] = hash_val, data
+ return True, data
+
+
+class WdspecExecutor(TestExecutor):
+ convert_result = pytest_result_converter
+
+
+class Protocol(object):
+ def __init__(self, executor, browser):
+ self.executor = executor
+ self.browser = browser
+
+ @property
+ def logger(self):
+ return self.executor.logger
+
+ def setup(self, runner):
+ pass
+
+ def teardown(self):
+ pass
+
+ def wait(self):
+ pass
diff --git a/testing/web-platform/harness/wptrunner/executors/executormarionette.py b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
new file mode 100644
index 000000000..e212c2945
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -0,0 +1,595 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+import httplib
+import os
+import socket
+import threading
+import time
+import traceback
+import urlparse
+import uuid
+from collections import defaultdict
+
+from ..wpttest import WdspecResult, WdspecSubtestResult
+
+errors = None
+marionette = None
+webdriver = None
+
+here = os.path.join(os.path.split(__file__)[0])
+
+from . import pytestrunner
+from .base import (ExecutorException,
+ Protocol,
+ RefTestExecutor,
+ RefTestImplementation,
+ TestExecutor,
+ TestharnessExecutor,
+ testharness_result_converter,
+ reftest_result_converter,
+ strip_server,
+ WdspecExecutor)
+from ..testrunner import Stop
+from ..webdriver_server import GeckoDriverServer
+
+# Extra timeout to use after internal test timeout at which the harness
+# should force a timeout
+extra_timeout = 5 # seconds
+
+
+def do_delayed_imports():
+ global errors, marionette, webdriver
+
+ # Marionette client used to be called marionette, recently it changed
+ # to marionette_driver for unfathomable reasons
+ try:
+ import marionette
+ from marionette import errors
+ except ImportError:
+ from marionette_driver import marionette, errors
+
+ import webdriver
+
+
+class MarionetteProtocol(Protocol):
+ def __init__(self, executor, browser):
+ do_delayed_imports()
+
+ Protocol.__init__(self, executor, browser)
+ self.marionette = None
+ self.marionette_port = browser.marionette_port
+ self.timeout = None
+ self.runner_handle = None
+
+ def setup(self, runner):
+ """Connect to browser via Marionette."""
+ Protocol.setup(self, runner)
+
+ self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
+ self.marionette = marionette.Marionette(host='localhost',
+ port=self.marionette_port,
+ socket_timeout=None)
+
+ # XXX Move this timeout somewhere
+ self.logger.debug("Waiting for Marionette connection")
+ while True:
+ success = self.marionette.wait_for_port(60)
+ #When running in a debugger wait indefinitely for firefox to start
+ if success or self.executor.debug_info is None:
+ break
+
+ session_started = False
+ if success:
+ try:
+ self.logger.debug("Starting Marionette session")
+ self.marionette.start_session()
+ except Exception as e:
+ self.logger.warning("Starting marionette session failed: %s" % e)
+ else:
+ self.logger.debug("Marionette session started")
+ session_started = True
+
+ if not success or not session_started:
+ self.logger.warning("Failed to connect to Marionette")
+ self.executor.runner.send_message("init_failed")
+ else:
+ try:
+ self.after_connect()
+ except Exception:
+ self.logger.warning("Post-connection steps failed")
+ self.logger.error(traceback.format_exc())
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ try:
+ self.marionette.delete_session()
+ except Exception:
+ # This is typically because the session never started
+ pass
+ del self.marionette
+
+ @property
+ def is_alive(self):
+ """Check if the Marionette connection is still active."""
+ try:
+ self.marionette.current_window_handle
+ except Exception:
+ return False
+ return True
+
+ def after_connect(self):
+ self.load_runner("http")
+
+ def set_timeout(self, timeout):
+ """Set the Marionette script timeout.
+
+ :param timeout: Script timeout in seconds
+
+ """
+ self.marionette.timeout.script = timeout
+ self.timeout = timeout
+
+ def load_runner(self, protocol):
+ # Check if we previously had a test window open, and if we did make sure it's closed
+ self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
+ url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
+ self.logger.debug("Loading %s" % url)
+ self.runner_handle = self.marionette.current_window_handle
+ try:
+ self.marionette.navigate(url)
+ except Exception as e:
+ self.logger.critical(
+ "Loading initial page %s failed. Ensure that the "
+ "there are no other programs bound to this port and "
+ "that your firewall rules or network setup does not "
+ "prevent access.\e%s" % (url, traceback.format_exc(e)))
+ self.marionette.execute_script(
+ "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
+
+ def close_old_windows(self, protocol):
+ handles = self.marionette.window_handles
+ runner_handle = None
+ try:
+ handles.remove(self.runner_handle)
+ runner_handle = self.runner_handle
+ except ValueError:
+ # The runner window probably changed id but we can restore it
+ # This isn't supposed to happen, but marionette ids are not yet stable
+ # We assume that the first handle returned corresponds to the runner,
+ # but it hopefully doesn't matter too much if that assumption is
+ # wrong since we reload the runner in that tab anyway.
+ runner_handle = handles.pop(0)
+
+ for handle in handles:
+ self.marionette.switch_to_window(handle)
+ self.marionette.close()
+
+ self.marionette.switch_to_window(runner_handle)
+ if runner_handle != self.runner_handle:
+ self.load_runner(protocol)
+
+ def wait(self):
+ socket_timeout = self.marionette.client.sock.gettimeout()
+ if socket_timeout:
+ self.marionette.timeout.script = socket_timeout / 2
+
+ while True:
+ try:
+ self.marionette.execute_async_script("")
+ except errors.ScriptTimeoutException:
+ self.logger.debug("Script timed out")
+ pass
+ except (socket.timeout, IOError):
+ self.logger.debug("Socket closed")
+ break
+ except Exception as e:
+ self.logger.error(traceback.format_exc(e))
+ break
+
+ def on_environment_change(self, old_environment, new_environment):
+ #Unset all the old prefs
+ for name in old_environment.get("prefs", {}).iterkeys():
+ value = self.executor.original_pref_values[name]
+ if value is None:
+ self.clear_user_pref(name)
+ else:
+ self.set_pref(name, value)
+
+ for name, value in new_environment.get("prefs", {}).iteritems():
+ self.executor.original_pref_values[name] = self.get_pref(name)
+ self.set_pref(name, value)
+
+ def set_pref(self, name, value):
+ if value.lower() not in ("true", "false"):
+ try:
+ int(value)
+ except ValueError:
+ value = "'%s'" % value
+ else:
+ value = value.lower()
+
+ self.logger.info("Setting pref %s (%s)" % (name, value))
+
+ script = """
+ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
+ .getService(Components.interfaces.nsIPrefBranch);
+ let pref = '%s';
+ let type = prefInterface.getPrefType(pref);
+ let value = %s;
+ switch(type) {
+ case prefInterface.PREF_STRING:
+ prefInterface.setCharPref(pref, value);
+ break;
+ case prefInterface.PREF_BOOL:
+ prefInterface.setBoolPref(pref, value);
+ break;
+ case prefInterface.PREF_INT:
+ prefInterface.setIntPref(pref, value);
+ break;
+ }
+ """ % (name, value)
+ with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+ self.marionette.execute_script(script)
+
+ def clear_user_pref(self, name):
+ self.logger.info("Clearing pref %s" % (name))
+ script = """
+ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
+ .getService(Components.interfaces.nsIPrefBranch);
+ let pref = '%s';
+ prefInterface.clearUserPref(pref);
+ """ % name
+ with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+ self.marionette.execute_script(script)
+
+ def get_pref(self, name):
+ script = """
+ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
+ .getService(Components.interfaces.nsIPrefBranch);
+ let pref = '%s';
+ let type = prefInterface.getPrefType(pref);
+ switch(type) {
+ case prefInterface.PREF_STRING:
+ return prefInterface.getCharPref(pref);
+ case prefInterface.PREF_BOOL:
+ return prefInterface.getBoolPref(pref);
+ case prefInterface.PREF_INT:
+ return prefInterface.getIntPref(pref);
+ case prefInterface.PREF_INVALID:
+ return null;
+ }
+ """ % name
+ with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+ self.marionette.execute_script(script)
+
+
+class RemoteMarionetteProtocol(Protocol):
+ def __init__(self, executor, browser):
+ do_delayed_imports()
+ Protocol.__init__(self, executor, browser)
+ self.session = None
+ self.webdriver_binary = executor.webdriver_binary
+ self.marionette_port = browser.marionette_port
+ self.server = None
+
+ def setup(self, runner):
+ """Connect to browser via the Marionette HTTP server."""
+ try:
+ self.server = GeckoDriverServer(
+ self.logger, self.marionette_port, binary=self.webdriver_binary)
+ self.server.start(block=False)
+ self.logger.info(
+ "WebDriver HTTP server listening at %s" % self.server.url)
+
+ self.logger.info(
+ "Establishing new WebDriver session with %s" % self.server.url)
+ self.session = webdriver.Session(
+ self.server.host, self.server.port, self.server.base_path)
+ except Exception:
+ self.logger.error(traceback.format_exc())
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ try:
+ if self.session.session_id is not None:
+ self.session.end()
+ except Exception:
+ pass
+ if self.server is not None and self.server.is_alive:
+ self.server.stop()
+
+ @property
+ def is_alive(self):
+ """Test that the Marionette connection is still alive.
+
+ Because the remote communication happens over HTTP we need to
+ make an explicit request to the remote. It is allowed for
+ WebDriver spec tests to not have a WebDriver session, since this
+ may be what is tested.
+
+ An HTTP request to an invalid path that results in a 404 is
+ proof enough to us that the server is alive and kicking.
+ """
+ conn = httplib.HTTPConnection(self.server.host, self.server.port)
+ conn.request("HEAD", self.server.base_path + "invalid")
+ res = conn.getresponse()
+ return res.status == 404
+
+
+class ExecuteAsyncScriptRun(object):
+ def __init__(self, logger, func, protocol, url, timeout):
+ self.logger = logger
+ self.result = (None, None)
+ self.protocol = protocol
+ self.marionette = protocol.marionette
+ self.func = func
+ self.url = url
+ self.timeout = timeout
+ self.result_flag = threading.Event()
+
+ def run(self):
+ timeout = self.timeout
+
+ try:
+ if timeout is not None:
+ if timeout + extra_timeout != self.protocol.timeout:
+ self.protocol.set_timeout(timeout + extra_timeout)
+ else:
+ # We just want it to never time out, really, but marionette doesn't
+ # make that possible. It also seems to time out immediately if the
+ # timeout is set too high. This works at least.
+ self.protocol.set_timeout(2**28 - 1)
+ except IOError:
+ self.logger.error("Lost marionette connection before starting test")
+ return Stop
+
+ executor = threading.Thread(target = self._run)
+ executor.start()
+
+ if timeout is not None:
+ wait_timeout = timeout + 2 * extra_timeout
+ else:
+ wait_timeout = None
+
+ flag = self.result_flag.wait(wait_timeout)
+ if self.result[1] is None:
+ self.logger.debug("Timed out waiting for a result")
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+ return self.result
+
+ def _run(self):
+ try:
+ self.result = True, self.func(self.marionette, self.url, self.timeout)
+ except errors.ScriptTimeoutException:
+ self.logger.debug("Got a marionette timeout")
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+ except (socket.timeout, IOError):
+ # This can happen on a crash
+ # Also, should check after the test if the firefox process is still running
+ # and otherwise ignore any other result and set it to crash
+ self.result = False, ("CRASH", None)
+ except Exception as e:
+ message = getattr(e, "message", "")
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ self.result = False, ("ERROR", e)
+
+ finally:
+ self.result_flag.set()
+
+
+class MarionetteTestharnessExecutor(TestharnessExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ close_after_done=True, debug_info=None, **kwargs):
+ """Marionette-based executor for testharness.js tests"""
+ TestharnessExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+
+ self.protocol = MarionetteProtocol(self, browser)
+ self.script = open(os.path.join(here, "testharness_marionette.js")).read()
+ self.close_after_done = close_after_done
+ self.window_id = str(uuid.uuid4())
+
+ self.original_pref_values = {}
+
+ if marionette is None:
+ do_delayed_imports()
+
+ def is_alive(self):
+ return self.protocol.is_alive
+
+ def on_environment_change(self, new_environment):
+ self.protocol.on_environment_change(self.last_environment, new_environment)
+
+ if new_environment["protocol"] != self.last_environment["protocol"]:
+ self.protocol.load_runner(new_environment["protocol"])
+
+ def do_test(self, test):
+ timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
+ else None)
+
+ success, data = ExecuteAsyncScriptRun(self.logger,
+ self.do_testharness,
+ self.protocol,
+ self.test_url(test),
+ timeout).run()
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_testharness(self, marionette, url, timeout):
+ if self.close_after_done:
+ marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
+ self.protocol.close_old_windows(self.protocol)
+
+ if timeout is not None:
+ timeout_ms = str(timeout * 1000)
+ else:
+ timeout_ms = "null"
+
+ script = self.script % {"abs_url": url,
+ "url": strip_server(url),
+ "window_id": self.window_id,
+ "timeout_multiplier": self.timeout_multiplier,
+ "timeout": timeout_ms,
+ "explicit_timeout": timeout is None}
+
+ rv = marionette.execute_async_script(script, new_sandbox=False)
+ return rv
+
+
+class MarionetteRefTestExecutor(RefTestExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ screenshot_cache=None, close_after_done=True,
+ debug_info=None, **kwargs):
+
+ """Marionette-based executor for reftests"""
+ RefTestExecutor.__init__(self,
+ browser,
+ server_config,
+ screenshot_cache=screenshot_cache,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = MarionetteProtocol(self, browser)
+ self.implementation = RefTestImplementation(self)
+ self.close_after_done = close_after_done
+ self.has_window = False
+ self.original_pref_values = {}
+
+ with open(os.path.join(here, "reftest.js")) as f:
+ self.script = f.read()
+ with open(os.path.join(here, "reftest-wait.js")) as f:
+ self.wait_script = f.read()
+
+ def is_alive(self):
+ return self.protocol.is_alive
+
+ def on_environment_change(self, new_environment):
+ self.protocol.on_environment_change(self.last_environment, new_environment)
+
+ def do_test(self, test):
+ if self.close_after_done and self.has_window:
+ self.protocol.marionette.close()
+ self.protocol.marionette.switch_to_window(
+ self.protocol.marionette.window_handles[-1])
+ self.has_window = False
+
+ if not self.has_window:
+ self.protocol.marionette.execute_script(self.script)
+ self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
+ self.has_window = True
+
+ result = self.implementation.run_test(test)
+ return self.convert_result(test, result)
+
+ def screenshot(self, test, viewport_size, dpi):
+ # https://github.com/w3c/wptrunner/issues/166
+ assert viewport_size is None
+ assert dpi is None
+
+ timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
+
+ test_url = self.test_url(test)
+
+ return ExecuteAsyncScriptRun(self.logger,
+ self._screenshot,
+ self.protocol,
+ test_url,
+ timeout).run()
+
+ def _screenshot(self, marionette, url, timeout):
+ marionette.navigate(url)
+
+ marionette.execute_async_script(self.wait_script)
+
+ screenshot = marionette.screenshot()
+ # strip off the data:img/png, part of the url
+ if screenshot.startswith("data:image/png;base64,"):
+ screenshot = screenshot.split(",", 1)[1]
+
+ return screenshot
+
+
+class WdspecRun(object):
+ def __init__(self, func, session, path, timeout):
+ self.func = func
+ self.result = (None, None)
+ self.session = session
+ self.path = path
+ self.timeout = timeout
+ self.result_flag = threading.Event()
+
+ def run(self):
+ """Runs function in a thread and interrupts it if it exceeds the
+ given timeout. Returns (True, (Result, [SubtestResult ...])) in
+ case of success, or (False, (status, extra information)) in the
+ event of failure.
+ """
+
+ executor = threading.Thread(target=self._run)
+ executor.start()
+
+ flag = self.result_flag.wait(self.timeout)
+ if self.result[1] is None:
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+
+ return self.result
+
+ def _run(self):
+ try:
+ self.result = True, self.func(self.session, self.path, self.timeout)
+ except (socket.timeout, IOError):
+ self.result = False, ("CRASH", None)
+ except Exception as e:
+ message = getattr(e, "message")
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ self.result = False, ("ERROR", message)
+ finally:
+ self.result_flag.set()
+
+
+class MarionetteWdspecExecutor(WdspecExecutor):
+ def __init__(self, browser, server_config, webdriver_binary,
+ timeout_multiplier=1, close_after_done=True, debug_info=None):
+ WdspecExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.webdriver_binary = webdriver_binary
+ self.protocol = RemoteMarionetteProtocol(self, browser)
+
+ def is_alive(self):
+ return self.protocol.is_alive
+
+ def on_environment_change(self, new_environment):
+ pass
+
+ def do_test(self, test):
+ timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+ success, data = WdspecRun(self.do_wdspec,
+ self.protocol.session,
+ test.abs_path,
+ timeout).run()
+
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_wdspec(self, session, path, timeout):
+ harness_result = ("OK", None)
+ subtest_results = pytestrunner.run(
+ path, session, self.server_url, timeout=timeout)
+ return (harness_result, subtest_results)
diff --git a/testing/web-platform/harness/wptrunner/executors/executorselenium.py b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
new file mode 100644
index 000000000..b8b209a77
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -0,0 +1,271 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import socket
+import sys
+import threading
+import time
+import traceback
+import urlparse
+import uuid
+
+from .base import (ExecutorException,
+ Protocol,
+ RefTestExecutor,
+ RefTestImplementation,
+ TestExecutor,
+ TestharnessExecutor,
+ testharness_result_converter,
+ reftest_result_converter,
+ strip_server)
+from ..testrunner import Stop
+
+
+here = os.path.join(os.path.split(__file__)[0])
+
+webdriver = None
+exceptions = None
+
+extra_timeout = 5
+
+def do_delayed_imports():
+ global webdriver
+ global exceptions
+ from selenium import webdriver
+ from selenium.common import exceptions
+
+
+class SeleniumProtocol(Protocol):
+ def __init__(self, executor, browser, capabilities, **kwargs):
+ do_delayed_imports()
+
+ Protocol.__init__(self, executor, browser)
+ self.capabilities = capabilities
+ self.url = browser.webdriver_url
+ self.webdriver = None
+
+ def setup(self, runner):
+ """Connect to browser via Selenium's WebDriver implementation."""
+ self.runner = runner
+ self.logger.debug("Connecting to Selenium on URL: %s" % self.url)
+
+ session_started = False
+ try:
+ self.webdriver = webdriver.Remote(
+ self.url, desired_capabilities=self.capabilities)
+ except:
+ self.logger.warning(
+ "Connecting to Selenium failed:\n%s" % traceback.format_exc())
+ else:
+ self.logger.debug("Selenium session started")
+ session_started = True
+
+ if not session_started:
+ self.logger.warning("Failed to connect to Selenium")
+ self.executor.runner.send_message("init_failed")
+ else:
+ try:
+ self.after_connect()
+ except:
+ print >> sys.stderr, traceback.format_exc()
+ self.logger.warning(
+ "Failed to connect to navigate initial page")
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ self.logger.debug("Hanging up on Selenium session")
+ try:
+ self.webdriver.quit()
+ except:
+ pass
+ del self.webdriver
+
+ def is_alive(self):
+ try:
+ # Get a simple property over the connection
+ self.webdriver.current_window_handle
+ # TODO what exception?
+ except (socket.timeout, exceptions.ErrorInResponseException):
+ return False
+ return True
+
+ def after_connect(self):
+ self.load_runner("http")
+
+ def load_runner(self, protocol):
+ url = urlparse.urljoin(self.executor.server_url(protocol),
+ "/testharness_runner.html")
+ self.logger.debug("Loading %s" % url)
+ self.webdriver.get(url)
+ self.webdriver.execute_script("document.title = '%s'" %
+ threading.current_thread().name.replace("'", '"'))
+
+ def wait(self):
+ while True:
+ try:
+ self.webdriver.execute_async_script("");
+ except exceptions.TimeoutException:
+ pass
+ except (socket.timeout, exceptions.NoSuchWindowException,
+ exceptions.ErrorInResponseException, IOError):
+ break
+ except Exception as e:
+ self.logger.error(traceback.format_exc(e))
+ break
+
+
+class SeleniumRun(object):
+ def __init__(self, func, webdriver, url, timeout):
+ self.func = func
+ self.result = None
+ self.webdriver = webdriver
+ self.url = url
+ self.timeout = timeout
+ self.result_flag = threading.Event()
+
+ def run(self):
+ timeout = self.timeout
+
+ try:
+ self.webdriver.set_script_timeout((timeout + extra_timeout) * 1000)
+ except exceptions.ErrorInResponseException:
+ self.logger.error("Lost WebDriver connection")
+ return Stop
+
+ executor = threading.Thread(target=self._run)
+ executor.start()
+
+ flag = self.result_flag.wait(timeout + 2 * extra_timeout)
+ if self.result is None:
+ assert not flag
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+
+ return self.result
+
+ def _run(self):
+ try:
+ self.result = True, self.func(self.webdriver, self.url, self.timeout)
+ except exceptions.TimeoutException:
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+ except (socket.timeout, exceptions.ErrorInResponseException):
+ self.result = False, ("CRASH", None)
+ except Exception as e:
+ message = getattr(e, "message", "")
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ self.result = False, ("ERROR", e)
+ finally:
+ self.result_flag.set()
+
+
+class SeleniumTestharnessExecutor(TestharnessExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ close_after_done=True, capabilities=None, debug_info=None):
+ """Selenium-based executor for testharness.js tests"""
+ TestharnessExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = SeleniumProtocol(self, browser, capabilities)
+ with open(os.path.join(here, "testharness_webdriver.js")) as f:
+ self.script = f.read()
+ self.close_after_done = close_after_done
+ self.window_id = str(uuid.uuid4())
+
+ def is_alive(self):
+ return self.protocol.is_alive()
+
+ def on_protocol_change(self, new_protocol):
+ self.protocol.load_runner(new_protocol)
+
+ def do_test(self, test):
+ url = self.test_url(test)
+
+ success, data = SeleniumRun(self.do_testharness,
+ self.protocol.webdriver,
+ url,
+ test.timeout * self.timeout_multiplier).run()
+
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_testharness(self, webdriver, url, timeout):
+ return webdriver.execute_async_script(
+ self.script % {"abs_url": url,
+ "url": strip_server(url),
+ "window_id": self.window_id,
+ "timeout_multiplier": self.timeout_multiplier,
+ "timeout": timeout * 1000})
+
+class SeleniumRefTestExecutor(RefTestExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ screenshot_cache=None, close_after_done=True,
+ debug_info=None, capabilities=None):
+ """Selenium WebDriver-based executor for reftests"""
+ RefTestExecutor.__init__(self,
+ browser,
+ server_config,
+ screenshot_cache=screenshot_cache,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = SeleniumProtocol(self, browser,
+ capabilities=capabilities)
+ self.implementation = RefTestImplementation(self)
+ self.close_after_done = close_after_done
+ self.has_window = False
+
+ with open(os.path.join(here, "reftest.js")) as f:
+ self.script = f.read()
+ with open(os.path.join(here, "reftest-wait_webdriver.js")) as f:
+ self.wait_script = f.read()
+
+ def is_alive(self):
+ return self.protocol.is_alive()
+
+ def do_test(self, test):
+ self.logger.info("Test requires OS-level window focus")
+
+ if self.close_after_done and self.has_window:
+ self.protocol.webdriver.close()
+ self.protocol.webdriver.switch_to_window(
+ self.protocol.webdriver.window_handles[-1])
+ self.has_window = False
+
+ if not self.has_window:
+ self.protocol.webdriver.execute_script(self.script)
+ self.protocol.webdriver.switch_to_window(
+ self.protocol.webdriver.window_handles[-1])
+ self.has_window = True
+
+ result = self.implementation.run_test(test)
+
+ return self.convert_result(test, result)
+
+ def screenshot(self, test, viewport_size, dpi):
+ # https://github.com/w3c/wptrunner/issues/166
+ assert viewport_size is None
+ assert dpi is None
+
+ return SeleniumRun(self._screenshot,
+ self.protocol.webdriver,
+ self.test_url(test),
+ test.timeout).run()
+
+ def _screenshot(self, webdriver, url, timeout):
+ webdriver.get(url)
+
+ webdriver.execute_async_script(self.wait_script)
+
+ screenshot = webdriver.get_screenshot_as_base64()
+
+ # strip off the data:img/png, part of the url
+ if screenshot.startswith("data:image/png;base64,"):
+ screenshot = screenshot.split(",", 1)[1]
+
+ return screenshot
diff --git a/testing/web-platform/harness/wptrunner/executors/executorservo.py b/testing/web-platform/harness/wptrunner/executors/executorservo.py
new file mode 100644
index 000000000..068061b95
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -0,0 +1,275 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import base64
+import hashlib
+import json
+import os
+import subprocess
+import tempfile
+import threading
+import urlparse
+import uuid
+from collections import defaultdict
+
+from mozprocess import ProcessHandler
+
+from .base import (ExecutorException,
+ Protocol,
+ RefTestImplementation,
+ testharness_result_converter,
+ reftest_result_converter)
+from .process import ProcessTestExecutor
+from ..browsers.base import browser_command
+render_arg = None
+
+
+def do_delayed_imports():
+ global render_arg
+ from ..browsers.servo import render_arg
+
+hosts_text = """127.0.0.1 web-platform.test
+127.0.0.1 www.web-platform.test
+127.0.0.1 www1.web-platform.test
+127.0.0.1 www2.web-platform.test
+127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
+127.0.0.1 xn--lve-6lad.web-platform.test
+"""
+
+def make_hosts_file():
+ hosts_fd, hosts_path = tempfile.mkstemp()
+ with os.fdopen(hosts_fd, "w") as f:
+ f.write(hosts_text)
+ return hosts_path
+
+
+class ServoTestharnessExecutor(ProcessTestExecutor):
+ convert_result = testharness_result_converter
+
+ def __init__(self, browser, server_config, timeout_multiplier=1, debug_info=None,
+ pause_after_test=False):
+ do_delayed_imports()
+ ProcessTestExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.pause_after_test = pause_after_test
+ self.result_data = None
+ self.result_flag = None
+ self.protocol = Protocol(self, browser)
+ self.hosts_path = make_hosts_file()
+
+ def teardown(self):
+ try:
+ os.unlink(self.hosts_path)
+ except OSError:
+ pass
+ ProcessTestExecutor.teardown(self)
+
+ def do_test(self, test):
+ self.result_data = None
+ self.result_flag = threading.Event()
+
+ args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner",
+ "-Z", "replace-surrogates", "-z", self.test_url(test)]
+ for stylesheet in self.browser.user_stylesheets:
+ args += ["--user-stylesheet", stylesheet]
+ for pref, value in test.environment.get('prefs', {}).iteritems():
+ args += ["--pref", "%s=%s" % (pref, value)]
+ args += self.browser.binary_args
+ debug_args, command = browser_command(self.binary, args, self.debug_info)
+
+ self.command = command
+
+ if self.pause_after_test:
+ self.command.remove("-z")
+
+ self.command = debug_args + self.command
+
+ env = os.environ.copy()
+ env["HOST_FILE"] = self.hosts_path
+ env["RUST_BACKTRACE"] = "1"
+
+
+ if not self.interactive:
+ self.proc = ProcessHandler(self.command,
+ processOutputLine=[self.on_output],
+ onFinish=self.on_finish,
+ env=env,
+ storeOutput=False)
+ self.proc.run()
+ else:
+ self.proc = subprocess.Popen(self.command, env=env)
+
+ try:
+ timeout = test.timeout * self.timeout_multiplier
+
+ # Now wait to get the output we expect, or until we reach the timeout
+ if not self.interactive and not self.pause_after_test:
+ wait_timeout = timeout + 5
+ self.result_flag.wait(wait_timeout)
+ else:
+ wait_timeout = None
+ self.proc.wait()
+
+ proc_is_running = True
+
+ if self.result_flag.is_set():
+ if self.result_data is not None:
+ result = self.convert_result(test, self.result_data)
+ else:
+ self.proc.wait()
+ result = (test.result_cls("CRASH", None), [])
+ proc_is_running = False
+ else:
+ result = (test.result_cls("TIMEOUT", None), [])
+
+
+ if proc_is_running:
+ if self.pause_after_test:
+ self.logger.info("Pausing until the browser exits")
+ self.proc.wait()
+ else:
+ self.proc.kill()
+ except KeyboardInterrupt:
+ self.proc.kill()
+ raise
+
+ return result
+
+ def on_output(self, line):
+ prefix = "ALERT: RESULT: "
+ line = line.decode("utf8", "replace")
+ if line.startswith(prefix):
+ self.result_data = json.loads(line[len(prefix):])
+ self.result_flag.set()
+ else:
+ if self.interactive:
+ print line
+ else:
+ self.logger.process_output(self.proc.pid,
+ line,
+ " ".join(self.command))
+
+ def on_finish(self):
+ self.result_flag.set()
+
+
+class TempFilename(object):
+ def __init__(self, directory):
+ self.directory = directory
+ self.path = None
+
+ def __enter__(self):
+ self.path = os.path.join(self.directory, str(uuid.uuid4()))
+ return self.path
+
+ def __exit__(self, *args, **kwargs):
+ try:
+ os.unlink(self.path)
+ except OSError:
+ pass
+
+
+class ServoRefTestExecutor(ProcessTestExecutor):
+ convert_result = reftest_result_converter
+
+ def __init__(self, browser, server_config, binary=None, timeout_multiplier=1,
+ screenshot_cache=None, debug_info=None, pause_after_test=False):
+ do_delayed_imports()
+ ProcessTestExecutor.__init__(self,
+ browser,
+ server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+
+ self.protocol = Protocol(self, browser)
+ self.screenshot_cache = screenshot_cache
+ self.implementation = RefTestImplementation(self)
+ self.tempdir = tempfile.mkdtemp()
+ self.hosts_path = make_hosts_file()
+
+ def teardown(self):
+ try:
+ os.unlink(self.hosts_path)
+ except OSError:
+ pass
+ os.rmdir(self.tempdir)
+ ProcessTestExecutor.teardown(self)
+
+ def screenshot(self, test, viewport_size, dpi):
+ full_url = self.test_url(test)
+
+ with TempFilename(self.tempdir) as output_path:
+ debug_args, command = browser_command(
+ self.binary,
+ [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
+ "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
+ "--output=%s" % output_path, full_url],
+ self.debug_info)
+
+ for stylesheet in self.browser.user_stylesheets:
+ command += ["--user-stylesheet", stylesheet]
+
+ for pref in test.environment.get('prefs', {}):
+ command += ["--pref", pref]
+
+ if viewport_size:
+ command += ["--resolution", viewport_size]
+
+ if dpi:
+ command += ["--device-pixel-ratio", dpi]
+
+ self.command = debug_args + command
+
+ env = os.environ.copy()
+ env["HOST_FILE"] = self.hosts_path
+ env["RUST_BACKTRACE"] = "1"
+
+ if not self.interactive:
+ self.proc = ProcessHandler(self.command,
+ processOutputLine=[self.on_output],
+ env=env)
+
+
+ try:
+ self.proc.run()
+ timeout = test.timeout * self.timeout_multiplier + 5
+ rv = self.proc.wait(timeout=timeout)
+ except KeyboardInterrupt:
+ self.proc.kill()
+ raise
+ else:
+ self.proc = subprocess.Popen(self.command,
+ env=env)
+ try:
+ rv = self.proc.wait()
+ except KeyboardInterrupt:
+ self.proc.kill()
+ raise
+
+ if rv is None:
+ self.proc.kill()
+ return False, ("EXTERNAL-TIMEOUT", None)
+
+ if rv != 0 or not os.path.exists(output_path):
+ return False, ("CRASH", None)
+
+ with open(output_path) as f:
+ # Might need to strip variable headers or something here
+ data = f.read()
+ return True, base64.b64encode(data)
+
+ def do_test(self, test):
+ result = self.implementation.run_test(test)
+
+ return self.convert_result(test, result)
+
+ def on_output(self, line):
+ line = line.decode("utf8", "replace")
+ if self.interactive:
+ print line
+ else:
+ self.logger.process_output(self.proc.pid,
+ line,
+ " ".join(self.command))
diff --git a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
new file mode 100644
index 000000000..fceeb58fa
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
@@ -0,0 +1,262 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import socket
+import threading
+import time
+import traceback
+
+from .base import (Protocol,
+ RefTestExecutor,
+ RefTestImplementation,
+ TestharnessExecutor,
+ strip_server)
+from .. import webdriver
+from ..testrunner import Stop
+
+webdriver = None
+
+here = os.path.join(os.path.split(__file__)[0])
+
+extra_timeout = 5
+
+
+def do_delayed_imports():
+ global webdriver
+ import webdriver
+
+
+class ServoWebDriverProtocol(Protocol):
+ def __init__(self, executor, browser, capabilities, **kwargs):
+ do_delayed_imports()
+ Protocol.__init__(self, executor, browser)
+ self.capabilities = capabilities
+ self.host = browser.webdriver_host
+ self.port = browser.webdriver_port
+ self.session = None
+
+ def setup(self, runner):
+ """Connect to browser via WebDriver."""
+ self.runner = runner
+
+ url = "http://%s:%d" % (self.host, self.port)
+ session_started = False
+ try:
+ self.session = webdriver.Session(self.host, self.port,
+ extension=webdriver.servo.ServoCommandExtensions)
+ self.session.start()
+ except:
+ self.logger.warning(
+ "Connecting with WebDriver failed:\n%s" % traceback.format_exc())
+ else:
+ self.logger.debug("session started")
+ session_started = True
+
+ if not session_started:
+ self.logger.warning("Failed to connect via WebDriver")
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ self.logger.debug("Hanging up on WebDriver session")
+ try:
+ self.session.end()
+ except:
+ pass
+
+ def is_alive(self):
+ try:
+ # Get a simple property over the connection
+ self.session.window_handle
+ # TODO what exception?
+ except Exception:
+ return False
+ return True
+
+ def after_connect(self):
+ pass
+
+ def wait(self):
+ while True:
+ try:
+ self.session.execute_async_script("")
+ except webdriver.TimeoutException:
+ pass
+ except (socket.timeout, IOError):
+ break
+ except Exception as e:
+ self.logger.error(traceback.format_exc(e))
+ break
+
+ def on_environment_change(self, old_environment, new_environment):
+ #Unset all the old prefs
+ self.session.extension.reset_prefs(*old_environment.get("prefs", {}).keys())
+ self.session.extension.set_prefs(new_environment.get("prefs", {}))
+
+
+class ServoWebDriverRun(object):
+ def __init__(self, func, session, url, timeout, current_timeout=None):
+ self.func = func
+ self.result = None
+ self.session = session
+ self.url = url
+ self.timeout = timeout
+ self.result_flag = threading.Event()
+
+ def run(self):
+ executor = threading.Thread(target=self._run)
+ executor.start()
+
+ flag = self.result_flag.wait(self.timeout + extra_timeout)
+ if self.result is None:
+ assert not flag
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+
+ return self.result
+
+ def _run(self):
+ try:
+ self.result = True, self.func(self.session, self.url, self.timeout)
+ except webdriver.TimeoutException:
+ self.result = False, ("EXTERNAL-TIMEOUT", None)
+ except (socket.timeout, IOError):
+ self.result = False, ("CRASH", None)
+ except Exception as e:
+ message = getattr(e, "message", "")
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ self.result = False, ("ERROR", e)
+ finally:
+ self.result_flag.set()
+
+
+def timeout_func(timeout):
+ if timeout:
+ t0 = time.time()
+ return lambda: time.time() - t0 > timeout + extra_timeout
+ else:
+ return lambda: False
+
+
+class ServoWebDriverTestharnessExecutor(TestharnessExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ close_after_done=True, capabilities=None, debug_info=None):
+ TestharnessExecutor.__init__(self, browser, server_config, timeout_multiplier=1,
+ debug_info=None)
+ self.protocol = ServoWebDriverProtocol(self, browser, capabilities=capabilities)
+ with open(os.path.join(here, "testharness_servodriver.js")) as f:
+ self.script = f.read()
+ self.timeout = None
+
+ def on_protocol_change(self, new_protocol):
+ pass
+
+ def is_alive(self):
+ return self.protocol.is_alive()
+
+ def do_test(self, test):
+ url = self.test_url(test)
+
+ timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+ if timeout != self.timeout:
+ try:
+ self.protocol.session.timeouts.script = timeout
+ self.timeout = timeout
+ except IOError:
+ self.logger.error("Lost webdriver connection")
+ return Stop
+
+ success, data = ServoWebDriverRun(self.do_testharness,
+ self.protocol.session,
+ url,
+ timeout).run()
+
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_testharness(self, session, url, timeout):
+ session.url = url
+ result = json.loads(
+ session.execute_async_script(
+ self.script % {"abs_url": url,
+ "url": strip_server(url),
+ "timeout_multiplier": self.timeout_multiplier,
+ "timeout": timeout * 1000}))
+ # Prevent leaking every page in history until Servo develops a more sane
+ # page cache
+ session.back()
+ return result
+
+
+class TimeoutError(Exception):
+ pass
+
+
+class ServoWebDriverRefTestExecutor(RefTestExecutor):
+ def __init__(self, browser, server_config, timeout_multiplier=1,
+ screenshot_cache=None, capabilities=None, debug_info=None):
+ """Selenium WebDriver-based executor for reftests"""
+ RefTestExecutor.__init__(self,
+ browser,
+ server_config,
+ screenshot_cache=screenshot_cache,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = ServoWebDriverProtocol(self, browser,
+ capabilities=capabilities)
+ self.implementation = RefTestImplementation(self)
+ self.timeout = None
+ with open(os.path.join(here, "reftest-wait_servodriver.js")) as f:
+ self.wait_script = f.read()
+
+ def is_alive(self):
+ return self.protocol.is_alive()
+
+ def do_test(self, test):
+ try:
+ result = self.implementation.run_test(test)
+ return self.convert_result(test, result)
+ except IOError:
+ return test.result_cls("CRASH", None), []
+ except TimeoutError:
+ return test.result_cls("TIMEOUT", None), []
+ except Exception as e:
+ message = getattr(e, "message", "")
+ if message:
+ message += "\n"
+ message += traceback.format_exc(e)
+ return test.result_cls("ERROR", message), []
+
+ def screenshot(self, test, viewport_size, dpi):
+ # https://github.com/w3c/wptrunner/issues/166
+ assert viewport_size is None
+ assert dpi is None
+
+ timeout = (test.timeout * self.timeout_multiplier + extra_timeout
+ if self.debug_info is None else None)
+
+ if self.timeout != timeout:
+ try:
+ self.protocol.session.timeouts.script = timeout
+ self.timeout = timeout
+ except IOError:
+ self.logger.error("Lost webdriver connection")
+ return Stop
+
+ return ServoWebDriverRun(self._screenshot,
+ self.protocol.session,
+ self.test_url(test),
+ timeout).run()
+
+ def _screenshot(self, session, url, timeout):
+ session.url = url
+ session.execute_async_script(self.wait_script)
+ return session.screenshot()
diff --git a/testing/web-platform/harness/wptrunner/executors/process.py b/testing/web-platform/harness/wptrunner/executors/process.py
new file mode 100644
index 000000000..45f33ab2c
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/process.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .base import TestExecutor
+
+
+class ProcessTestExecutor(TestExecutor):
+ def __init__(self, *args, **kwargs):
+ TestExecutor.__init__(self, *args, **kwargs)
+ self.binary = self.browser.binary
+ self.interactive = (False if self.debug_info is None
+ else self.debug_info.interactive)
+
+ def setup(self, runner):
+ self.runner = runner
+ self.runner.send_message("init_succeeded")
+ return True
+
+ def is_alive(self):
+ return True
+
+ def do_test(self, test):
+ raise NotImplementedError
diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py
new file mode 100644
index 000000000..de3a34a79
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py
@@ -0,0 +1,6 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from . import fixtures
+from .runner import run
diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py
new file mode 100644
index 000000000..1b4e8d43d
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pytest
+
+import urlparse
+
+
+"""pytest fixtures for use in Python-based WPT tests.
+
+The purpose of test fixtures is to provide a fixed baseline upon which
+tests can reliably and repeatedly execute.
+"""
+
+
+class Session(object):
+ """Fixture to allow access to wptrunner's existing WebDriver session
+ in tests.
+
+ The session is not created by default to enable testing of session
+ creation. However, a module-scoped session will be implicitly created
+ at the first call to a WebDriver command. This means methods such as
+ `session.send_command` and `session.session_id` are possible to use
+ without having a session.
+
+ To illustrate implicit session creation::
+
+ def test_session_scope(session):
+ # at this point there is no session
+ assert session.session_id is None
+
+ # window_id is a WebDriver command,
+ # and implicitly creates the session for us
+ assert session.window_id is not None
+
+ # we now have a session
+ assert session.session_id is not None
+
+ You can also access the session in custom fixtures defined in the
+ tests, such as a setup function::
+
+ @pytest.fixture(scope="function")
+ def setup(request, session):
+ session.url = "https://example.org"
+
+ def test_something(setup, session):
+ assert session.url == "https://example.org"
+
+ The session is closed when the test module goes out of scope by an
+ implicit call to `session.end`.
+ """
+
+ def __init__(self, client):
+ self.client = client
+
+ @pytest.fixture(scope="module")
+ def session(self, request):
+ request.addfinalizer(self.client.end)
+ return self.client
+
+class Server(object):
+ """Fixture to allow access to wptrunner's base server url.
+
+ :param url_getter: Function to get server url from test environment, given
+ a protocol.
+ """
+ def __init__(self, url_getter):
+ self.server_url = url_getter
+
+ def where_is(self, uri, protocol="http"):
+ return urlparse.urljoin(self.server_url(protocol), uri)
+
+ @pytest.fixture
+ def server(self, request):
+ return self
diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
new file mode 100644
index 000000000..28b8f609c
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
@@ -0,0 +1,116 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Provides interface to deal with pytest.
+
+Usage::
+
+ session = webdriver.client.Session("127.0.0.1", "4444", "/")
+ harness_result = ("OK", None)
+ subtest_results = pytestrunner.run("/path/to/test", session.url)
+ return (harness_result, subtest_results)
+"""
+
+import errno
+import shutil
+import tempfile
+
+from . import fixtures
+
+
+pytest = None
+
+
+def do_delayed_imports():
+ global pytest
+ import pytest
+
+
+def run(path, session, url_getter, timeout=0):
+ """Run Python test at ``path`` in pytest. The provided ``session``
+ is exposed as a fixture available in the scope of the test functions.
+
+ :param path: Path to the test file.
+ :param session: WebDriver session to expose.
+ :param url_getter: Function to get server url from test environment, given
+ a protocol.
+ :param timeout: Duration before interrupting potentially hanging
+ tests. If 0, there is no timeout.
+
+ :returns: List of subtest results, which are tuples of (test id,
+ status, message, stacktrace).
+ """
+
+ if pytest is None:
+ do_delayed_imports()
+
+ recorder = SubtestResultRecorder()
+ plugins = [recorder,
+ fixtures.Session(session),
+ fixtures.Server(url_getter)]
+
+ # TODO(ato): Deal with timeouts
+
+ with TemporaryDirectory() as cache:
+ pytest.main(["--strict", # turn warnings into errors
+ "--verbose", # show each individual subtest
+ "--capture", "no", # enable stdout/stderr from tests
+ "--basetemp", cache, # temporary directory
+ path],
+ plugins=plugins)
+
+ return recorder.results
+
+
+class SubtestResultRecorder(object):
+ def __init__(self):
+ self.results = []
+
+ def pytest_runtest_logreport(self, report):
+ if report.passed and report.when == "call":
+ self.record_pass(report)
+ elif report.failed:
+ if report.when != "call":
+ self.record_error(report)
+ else:
+ self.record_fail(report)
+ elif report.skipped:
+ self.record_skip(report)
+
+ def record_pass(self, report):
+ self.record(report.nodeid, "PASS")
+
+ def record_fail(self, report):
+ self.record(report.nodeid, "FAIL", stack=report.longrepr)
+
+ def record_error(self, report):
+ # error in setup/teardown
+ if report.when != "call":
+ message = "%s error" % report.when
+ self.record(report.nodeid, "ERROR", message, report.longrepr)
+
+ def record_skip(self, report):
+ self.record(report.nodeid, "ERROR",
+ "In-test skip decorators are disallowed, "
+ "please use WPT metadata to ignore tests.")
+
+ def record(self, test, status, message=None, stack=None):
+ if stack is not None:
+ stack = str(stack)
+ new_result = (test, status, message, stack)
+ self.results.append(new_result)
+
+
+class TemporaryDirectory(object):
+ def __enter__(self):
+ self.path = tempfile.mkdtemp(prefix="pytest-")
+ return self.path
+
+ def __exit__(self, *args):
+ try:
+ shutil.rmtree(self.path)
+ except OSError as e:
+ # no such file or directory
+ if e.errno != errno.ENOENT:
+ raise
diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait.js
new file mode 100644
index 000000000..a2bc66565
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait.js
@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function test(x) {
+ log("classList: " + root.classList);
+ if (!root.classList.contains("reftest-wait")) {
+ observer.disconnect();
+ marionetteScriptFinished();
+ }
+}
+
+var root = document.documentElement;
+var observer = new MutationObserver(test);
+
+observer.observe(root, {attributes: true});
+
+if (document.readyState != "complete") {
+ onload = test
+} else {
+ test();
+}
diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js
new file mode 100644
index 000000000..6040b4336
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js
@@ -0,0 +1,20 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+callback = arguments[arguments.length - 1];
+
+function check_done() {
+ if (!document.body.classList.contains('reftest-wait')) {
+ callback();
+ } else {
+ setTimeout(check_done, 50);
+ }
+}
+
+if (document.readyState === 'complete') {
+ check_done();
+} else {
+ addEventListener("load", check_done);
+}
diff --git a/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js b/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js
new file mode 100644
index 000000000..187f5daac
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait_webdriver.js
@@ -0,0 +1,23 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var callback = arguments[arguments.length - 1];
+
+function test(x) {
+ if (!root.classList.contains("reftest-wait")) {
+ observer.disconnect();
+ callback()
+ }
+}
+
+var root = document.documentElement;
+var observer = new MutationObserver(test);
+
+observer.observe(root, {attributes: true});
+
+if (document.readyState != "complete") {
+ onload = test;
+} else {
+ test();
+}
diff --git a/testing/web-platform/harness/wptrunner/executors/reftest.js b/testing/web-platform/harness/wptrunner/executors/reftest.js
new file mode 100644
index 000000000..3d7a9b6d2
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/reftest.js
@@ -0,0 +1,5 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var win = window.open("about:blank", "test", "width=600,height=600");
diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js b/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js
new file mode 100644
index 000000000..0d0e4017a
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js
@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+window.wrappedJSObject.timeout_multiplier = %(timeout_multiplier)d;
+window.wrappedJSObject.explicit_timeout = %(explicit_timeout)d;
+
+window.wrappedJSObject.addEventListener("message", function listener(event) {
+ if (event.data.type != "complete") {
+ return;
+ }
+ window.wrappedJSObject.removeEventListener("message", listener);
+ clearTimeout(timer);
+ var tests = event.data.tests;
+ var status = event.data.status;
+
+ var subtest_results = tests.map(function (x) {
+ return [x.name, x.status, x.message, x.stack]
+ });
+
+ marionetteScriptFinished(["%(url)s",
+ status.status,
+ status.message,
+ status.stack,
+ subtest_results]);
+}, false);
+
+window.wrappedJSObject.win = window.open("%(abs_url)s", "%(window_id)s");
+
+var timer = null;
+if (%(timeout)s) {
+ timer = setTimeout(function() {
+ log("Timeout fired");
+ window.wrappedJSObject.win.timeout();
+ }, %(timeout)s);
+}
diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js b/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js
new file mode 100644
index 000000000..20aeb34b9
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_servodriver.js
@@ -0,0 +1,6 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+window.__wd_results_callback__ = arguments[arguments.length - 1];
+window.__wd_results_timer__ = setTimeout(timeout, %(timeout)s);
diff --git a/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
new file mode 100644
index 000000000..7ca8d5737
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var callback = arguments[arguments.length - 1];
+window.timeout_multiplier = %(timeout_multiplier)d;
+
+window.addEventListener("message", function(event) {
+ var tests = event.data[0];
+ var status = event.data[1];
+
+ var subtest_results = tests.map(function(x) {
+ return [x.name, x.status, x.message, x.stack]
+ });
+
+ clearTimeout(timer);
+ callback(["%(url)s",
+ status.status,
+ status.message,
+ status.stack,
+ subtest_results]);
+}, false);
+
+window.win = window.open("%(abs_url)s", "%(window_id)s");
+
+var timer = setTimeout(function() {
+ window.win.timeout();
+ window.win.close();
+}, %(timeout)s);