diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/executors/pytestrunner')
3 files changed, 198 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py new file mode 100644 index 000000000..de3a34a79 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from . import fixtures +from .runner import run diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py new file mode 100644 index 000000000..1b4e8d43d --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +import urlparse + + +"""pytest fixtures for use in Python-based WPT tests. + +The purpose of test fixtures is to provide a fixed baseline upon which +tests can reliably and repeatedly execute. +""" + + +class Session(object): + """Fixture to allow access to wptrunner's existing WebDriver session + in tests. + + The session is not created by default to enable testing of session + creation. However, a module-scoped session will be implicitly created + at the first call to a WebDriver command. This means methods such as + `session.send_command` and `session.session_id` are possible to use + without having a session. + + To illustrate implicit session creation:: + + def test_session_scope(session): + # at this point there is no session + assert session.session_id is None + + # window_id is a WebDriver command, + # and implicitly creates the session for us + assert session.window_id is not None + + # we now have a session + assert session.session_id is not None + + You can also access the session in custom fixtures defined in the + tests, such as a setup function:: + + @pytest.fixture(scope="function") + def setup(request, session): + session.url = "https://example.org" + + def test_something(setup, session): + assert session.url == "https://example.org" + + The session is closed when the test module goes out of scope by an + implicit call to `session.end`. + """ + + def __init__(self, client): + self.client = client + + @pytest.fixture(scope="module") + def session(self, request): + request.addfinalizer(self.client.end) + return self.client + +class Server(object): + """Fixture to allow access to wptrunner's base server url. + + :param url_getter: Function to get server url from test environment, given + a protocol. + """ + def __init__(self, url_getter): + self.server_url = url_getter + + def where_is(self, uri, protocol="http"): + return urlparse.urljoin(self.server_url(protocol), uri) + + @pytest.fixture + def server(self, request): + return self diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py new file mode 100644 index 000000000..28b8f609c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Provides interface to deal with pytest. + +Usage:: + + session = webdriver.client.Session("127.0.0.1", "4444", "/") + harness_result = ("OK", None) + subtest_results = pytestrunner.run("/path/to/test", session.url) + return (harness_result, subtest_results) +""" + +import errno +import shutil +import tempfile + +from . import fixtures + + +pytest = None + + +def do_delayed_imports(): + global pytest + import pytest + + +def run(path, session, url_getter, timeout=0): + """Run Python test at ``path`` in pytest. The provided ``session`` + is exposed as a fixture available in the scope of the test functions. + + :param path: Path to the test file. + :param session: WebDriver session to expose. + :param url_getter: Function to get server url from test environment, given + a protocol. + :param timeout: Duration before interrupting potentially hanging + tests. If 0, there is no timeout. + + :returns: List of subtest results, which are tuples of (test id, + status, message, stacktrace). + """ + + if pytest is None: + do_delayed_imports() + + recorder = SubtestResultRecorder() + plugins = [recorder, + fixtures.Session(session), + fixtures.Server(url_getter)] + + # TODO(ato): Deal with timeouts + + with TemporaryDirectory() as cache: + pytest.main(["--strict", # turn warnings into errors + "--verbose", # show each individual subtest + "--capture", "no", # enable stdout/stderr from tests + "--basetemp", cache, # temporary directory + path], + plugins=plugins) + + return recorder.results + + +class SubtestResultRecorder(object): + def __init__(self): + self.results = [] + + def pytest_runtest_logreport(self, report): + if report.passed and report.when == "call": + self.record_pass(report) + elif report.failed: + if report.when != "call": + self.record_error(report) + else: + self.record_fail(report) + elif report.skipped: + self.record_skip(report) + + def record_pass(self, report): + self.record(report.nodeid, "PASS") + + def record_fail(self, report): + self.record(report.nodeid, "FAIL", stack=report.longrepr) + + def record_error(self, report): + # error in setup/teardown + if report.when != "call": + message = "%s error" % report.when + self.record(report.nodeid, "ERROR", message, report.longrepr) + + def record_skip(self, report): + self.record(report.nodeid, "ERROR", + "In-test skip decorators are disallowed, " + "please use WPT metadata to ignore tests.") + + def record(self, test, status, message=None, stack=None): + if stack is not None: + stack = str(stack) + new_result = (test, status, message, stack) + self.results.append(new_result) + + +class TemporaryDirectory(object): + def __enter__(self): + self.path = tempfile.mkdtemp(prefix="pytest-") + return self.path + + def __exit__(self, *args): + try: + shutil.rmtree(self.path) + except OSError as e: + # no such file or directory + if e.errno != errno.ENOENT: + raise |