diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/tests')
4 files changed, 463 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/tests/__init__.py b/testing/web-platform/harness/wptrunner/tests/__init__.py new file mode 100644 index 000000000..c580d191c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/tests/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/web-platform/harness/wptrunner/tests/test_chunker.py b/testing/web-platform/harness/wptrunner/tests/test_chunker.py new file mode 100644 index 000000000..d46ad2587 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/tests/test_chunker.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import sys +sys.path.insert(0, "..") + +from wptrunner import wptrunner + +class MockTest(object): + def __init__(self, id, timeout=10): + self.id = id + self.item_type = "testharness" + self.timeout = timeout + + +def make_mock_manifest(*items): + rv = [] + for dir_path, num_tests in items: + for i in range(num_tests): + rv.append((dir_path + "/%i.test" % i, set([MockTest(i)]))) + return rv + + +class TestEqualTimeChunker(unittest.TestCase): + + def test_include_all(self): + tests = make_mock_manifest(("a", 10), ("a/b", 10), ("c", 10)) + + chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests)) + chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests)) + chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests)) + + self.assertEquals(tests[:10], chunk_1) + self.assertEquals(tests[10:20], chunk_2) + self.assertEquals(tests[20:], chunk_3) + + def test_include_all_1(self): + tests = make_mock_manifest(("a", 5), ("a/b", 5), ("c", 10), ("d", 10)) + + chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests)) + chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests)) + chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests)) + + self.assertEquals(tests[:10], chunk_1) + self.assertEquals(tests[10:20], chunk_2) + self.assertEquals(tests[20:], chunk_3) + + def test_long(self): + tests = make_mock_manifest(("a", 100), ("a/b", 1), ("c", 1)) + + chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests)) + chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests)) + chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests)) + + self.assertEquals(tests[:100], chunk_1) + self.assertEquals(tests[100:101], chunk_2) + self.assertEquals(tests[101:102], chunk_3) + + def test_long_1(self): + tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1)) + + chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests)) + chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests)) + chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests)) + + self.assertEquals(tests[:1], chunk_1) + self.assertEquals(tests[1:101], chunk_2) + self.assertEquals(tests[101:102], chunk_3) + + def test_too_few_dirs(self): + with self.assertRaises(ValueError): + tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1)) + list(wptrunner.EqualTimeChunker(4, 1)(tests)) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/web-platform/harness/wptrunner/tests/test_hosts.py b/testing/web-platform/harness/wptrunner/tests/test_hosts.py new file mode 100644 index 000000000..4f122640d --- /dev/null +++ b/testing/web-platform/harness/wptrunner/tests/test_hosts.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import sys +from cStringIO import StringIO + +sys.path.insert(0, "..") + +import hosts + + +class HostsTest(unittest.TestCase): + def do_test(self, input, expected): + host_file = hosts.HostsFile.from_file(StringIO(input)) + self.assertEquals(host_file.to_string(), expected) + + def test_simple(self): + self.do_test("""127.0.0.1 \tlocalhost alias # comment +# Another comment""", + """127.0.0.1 localhost alias # comment +# Another comment +""") + + def test_blank_lines(self): + self.do_test("""127.0.0.1 \tlocalhost alias # comment + +\r + \t +# Another comment""", + """127.0.0.1 localhost alias # comment +# Another comment +""") + + def test_whitespace(self): + self.do_test(""" \t127.0.0.1 \tlocalhost alias # comment \r + \t# Another comment""", + """127.0.0.1 localhost alias # comment +# Another comment +""") + + def test_alignment(self): + self.do_test("""127.0.0.1 \tlocalhost alias +192.168.1.1 another_host another_alias +""","""127.0.0.1 localhost alias +192.168.1.1 another_host another_alias +""" +) + + def test_multiple_same_name(self): + # The semantics are that we overwrite earlier entries with the same name + self.do_test("""127.0.0.1 \tlocalhost alias +192.168.1.1 localhost another_alias""","""192.168.1.1 localhost another_alias +""" +) + +if __name__ == "__main__": + unittest.main() diff --git a/testing/web-platform/harness/wptrunner/tests/test_update.py b/testing/web-platform/harness/wptrunner/tests/test_update.py new file mode 100644 index 000000000..2d77b326d --- /dev/null +++ b/testing/web-platform/harness/wptrunner/tests/test_update.py @@ -0,0 +1,322 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import StringIO + +from .. import metadata, manifestupdate +from mozlog import structuredlog, handlers, formatters + + +class TestExpectedUpdater(unittest.TestCase): + def create_manifest(self, data, test_path="path/to/test.ini"): + f = StringIO.StringIO(data) + return manifestupdate.compile(f, test_path) + + def create_updater(self, data, **kwargs): + expected_tree = {} + id_path_map = {} + for test_path, test_ids, manifest_str in data: + if isinstance(test_ids, (str, unicode)): + test_ids = [test_ids] + expected_tree[test_path] = self.create_manifest(manifest_str, test_path) + for test_id in test_ids: + id_path_map[test_id] = test_path + + return metadata.ExpectedUpdater(expected_tree, id_path_map, **kwargs) + + def create_log(self, *args, **kwargs): + logger = structuredlog.StructuredLogger("expected_test") + data = StringIO.StringIO() + handler = handlers.StreamHandler(data, formatters.JSONFormatter()) + logger.add_handler(handler) + + log_entries = ([("suite_start", {"tests": [], "run_info": kwargs.get("run_info", {})})] + + list(args) + + [("suite_end", {})]) + + for item in log_entries: + action, kwargs = item + getattr(logger, action)(**kwargs) + logger.remove_handler(handler) + data.seek(0) + return data + + + def coalesce_results(self, trees): + for tree in trees: + for test in tree.iterchildren(): + for subtest in test.iterchildren(): + subtest.coalesce_expected() + test.coalesce_expected() + + def test_update_0(self): + prev_data = [("path/to/test.htm.ini", ["/path/to/test.htm"], """[test.htm] + type: testharness + [test1] + expected: FAIL""")] + + new_data = self.create_log(("test_start", {"test": "/path/to/test.htm"}), + ("test_status", {"test": "/path/to/test.htm", + "subtest": "test1", + "status": "PASS", + "expected": "FAIL"}), + ("test_end", {"test": "/path/to/test.htm", + "status": "OK"})) + updater = self.create_updater(prev_data) + updater.update_from_log(new_data) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + self.coalesce_results([new_manifest]) + self.assertTrue(new_manifest.is_empty) + + def test_update_1(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: ERROR""")] + + new_data = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "ERROR"}), + ("test_end", {"test": test_id, + "status": "OK"})) + updater = self.create_updater(prev_data) + updater.update_from_log(new_data) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + self.coalesce_results([new_manifest]) + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get("expected"), "FAIL") + + def test_new_subtest(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: FAIL""")] + + new_data = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "FAIL"}), + ("test_status", {"test": test_id, + "subtest": "test2", + "status": "FAIL", + "expected": "PASS"}), + ("test_end", {"test": test_id, + "status": "OK"})) + updater = self.create_updater(prev_data) + updater.update_from_log(new_data) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + self.coalesce_results([new_manifest]) + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get("expected"), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[1].get("expected"), "FAIL") + + def test_update_multiple_0(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: FAIL""")] + + new_data_0 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "osx"}) + + new_data_1 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "TIMEOUT", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "linux"}) + updater = self.create_updater(prev_data) + + updater.update_from_log(new_data_0) + updater.update_from_log(new_data_1) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + + self.coalesce_results([new_manifest]) + + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "osx"}), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "linux"}), "TIMEOUT") + + def test_update_multiple_1(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: FAIL""")] + + new_data_0 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "osx"}) + + new_data_1 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "TIMEOUT", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "linux"}) + updater = self.create_updater(prev_data) + + updater.update_from_log(new_data_0) + updater.update_from_log(new_data_1) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + + self.coalesce_results([new_manifest]) + + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "osx"}), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "linux"}), "TIMEOUT") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "windows"}), "FAIL") + + def test_update_multiple_2(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: FAIL""")] + + new_data_0 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "osx"}) + + new_data_1 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "TIMEOUT", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": True, "os": "osx"}) + updater = self.create_updater(prev_data) + + updater.update_from_log(new_data_0) + updater.update_from_log(new_data_1) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + + self.coalesce_results([new_manifest]) + + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "osx"}), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": True, "os": "osx"}), "TIMEOUT") + + def test_update_multiple_3(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: + if debug: FAIL + if not debug and os == "osx": TIMEOUT""")] + + new_data_0 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "osx"}) + + new_data_1 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "TIMEOUT", + "expected": "FAIL"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": True, "os": "osx"}) + updater = self.create_updater(prev_data) + + updater.update_from_log(new_data_0) + updater.update_from_log(new_data_1) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + + self.coalesce_results([new_manifest]) + + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "osx"}), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": True, "os": "osx"}), "TIMEOUT") + + def test_update_ignore_existing(self): + test_id = "/path/to/test.htm" + prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm] + type: testharness + [test1] + expected: + if debug: TIMEOUT + if not debug and os == "osx": NOTRUN""")] + + new_data_0 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "PASS"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": False, "os": "linux"}) + + new_data_1 = self.create_log(("test_start", {"test": test_id}), + ("test_status", {"test": test_id, + "subtest": "test1", + "status": "FAIL", + "expected": "PASS"}), + ("test_end", {"test": test_id, + "status": "OK"}), + run_info={"debug": True, "os": "windows"}) + updater = self.create_updater(prev_data, ignore_existing=True) + + updater.update_from_log(new_data_0) + updater.update_from_log(new_data_1) + + new_manifest = updater.expected_tree["path/to/test.htm.ini"] + + self.coalesce_results([new_manifest]) + + self.assertFalse(new_manifest.is_empty) + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": True, "os": "osx"}), "FAIL") + self.assertEquals(new_manifest.get_test(test_id).children[0].get( + "expected", {"debug": False, "os": "osx"}), "FAIL") |