diff options
Diffstat (limited to 'testing/web-platform/tests/tools/runner/runner.js')
| -rw-r--r-- | testing/web-platform/tests/tools/runner/runner.js | 854 |
1 files changed, 854 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/runner/runner.js b/testing/web-platform/tests/tools/runner/runner.js new file mode 100644 index 000000000..4319c234e --- /dev/null +++ b/testing/web-platform/tests/tools/runner/runner.js @@ -0,0 +1,854 @@ +/*jshint nonew: false */ +(function() { +"use strict"; +var runner; +var testharness_properties = {output:false, + timeout_multiplier:1}; + +function Manifest(path) { + this.data = null; + this.path = path; + this.num_tests = null; +} + +Manifest.prototype = { + load: function(loaded_callback) { + this.generate(loaded_callback); + }, + + do_load: function(loaded_callback) { + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState !== 4) { + return; + } + if (!(xhr.status === 200 || xhr.status === 0)) { + throw new Error("Manifest " + this.path + " failed to load"); + } + this.data = JSON.parse(xhr.responseText); + loaded_callback(); + }.bind(this); + xhr.open("GET", this.path); + xhr.send(null); + }, + + generate: function(loaded_callback) { + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState !== 4) { + return; + } + if (!(xhr.status === 200 || xhr.status === 0)) { + throw new Error("Manifest generation failed"); + } + this.do_load(loaded_callback); + }.bind(this); + xhr.open("POST", "update_manifest.py"); + xhr.send(null); + }, + + by_type:function(type) { + var ret = [] ; + if (this.data.items.hasOwnProperty(type)) { + ret = this.data.items[type].slice(0) ; + } + // local_changes.items in manifest is an Object just as + // items is. However, the properties of local_changes.items + // are Objects and the properties of items are Arrays. + // So we need to extract any relevant local changes by iterating + // over the Object and pulling out the referenced nodes as array items. + if (this.data.hasOwnProperty("local_changes")) { + var local = this.data.local_changes ; + // add in any local items + if (local.items.hasOwnProperty(type)) { + Object.keys(local.items[type]).forEach(function(ref) { + ret.push(local.items[type][ref][0]) ; + }.bind(this)); + } + // remove any items that are locally deleted but not yet committed + // note that the deleted and deleted_reftests properties of the local_changes + // object are always present, even if they are empty + if (ret.length && local.deleted.length) { + // make a hash of the deleted to speed searching + var dels = {} ; + local.deleted.forEach(function(x) { dels[x] = true; } ); + for (var j = ret.length-1; j >= 0; j--) { + if ( dels[ret[j].path] || (type === "reftest" && local.deleted_reftests[ret[j].path]) ){ + // we have a match + ret.splice(j, 1) ; + } + } + } + } + return ret ; + } +}; + +function ManifestIterator(manifest, path, test_types, use_regex) { + this.manifest = manifest; + this.paths = null; + this.regex_pattern = null; + this.test_types = test_types; + this.test_types_index = -1; + this.test_list = null; + this.test_index = null; + + if (use_regex) { + this.regex_pattern = path; + } else { + // Split paths by either a comma or whitespace, and ignore empty sub-strings. + this.paths = path.split(/[,\s]+/).filter(function(s) { return s.length > 0; }); + } +} + +ManifestIterator.prototype = { + next: function() { + var manifest_item = null; + + if (this.test_types.length === 0) { + return null; + } + + while (!manifest_item) { + while (this.test_list === null || this.test_index >= this.test_list.length) { + this.test_types_index++; + if (this.test_types_index >= this.test_types.length) { + return null; + } + this.test_index = 0; + this.test_list = this.manifest.by_type(this.test_types[this.test_types_index]); + } + + manifest_item = this.test_list[this.test_index++]; + while (manifest_item && !this.matches(manifest_item)) { + manifest_item = this.test_list[this.test_index++]; + } + if (manifest_item) { + return this.to_test(manifest_item); + } + } + }, + + matches: function(manifest_item) { + if (this.regex_pattern !== null) { + return manifest_item.url.match(this.regex_pattern); + } else { + return this.paths.some(function(p) { + return manifest_item.url.indexOf(p) === 0; + }); + } + }, + + to_test: function(manifest_item) { + var test = { + type: this.test_types[this.test_types_index], + url: manifest_item.url + }; + if (manifest_item.hasOwnProperty("references")) { + test.ref_length = manifest_item.references.length; + test.ref_type = manifest_item.references[0][1]; + test.ref_url = manifest_item.references[0][0]; + } + return test; + }, + + count: function() { + return this.test_types.reduce(function(prev, current) { + var matches = this.manifest.by_type(current).filter(function(x) { + return this.matches(x); + }.bind(this)); + return prev + matches.length; + }.bind(this), 0); + } +}; + +function VisualOutput(elem, runner) { + this.elem = elem; + this.runner = runner; + this.results_table = null; + this.section_wrapper = null; + this.results_table = this.elem.querySelector(".results > table"); + this.section = null; + this.manifest_status = this.elem.querySelector("#manifest"); + this.progress = this.elem.querySelector(".summary .progress"); + this.meter = this.progress.querySelector(".progress-bar"); + this.result_count = null; + this.json_results_area = this.elem.querySelector("textarea"); + this.instructions = document.querySelector(".instructions"); + + this.elem.style.display = "none"; + this.runner.manifest_wait_callbacks.push(this.on_manifest_wait.bind(this)); + this.runner.start_callbacks.push(this.on_start.bind(this)); + this.runner.result_callbacks.push(this.on_result.bind(this)); + this.runner.done_callbacks.push(this.on_done.bind(this)); + + this.display_filter_state = {}; + + var visual_output = this; + var display_filter_inputs = this.elem.querySelectorAll(".result-display-filter"); + for (var i = 0; i < display_filter_inputs.length; ++i) { + var display_filter_input = display_filter_inputs[i]; + this.display_filter_state[display_filter_input.value] = display_filter_input.checked; + display_filter_input.addEventListener("change", function(e) { + visual_output.apply_display_filter(e.target.value, e.target.checked); + }) + } +} + +VisualOutput.prototype = { + clear: function() { + this.result_count = {"PASS":0, + "FAIL":0, + "ERROR":0, + "TIMEOUT":0, + "NOTRUN":0}; + for (var p in this.result_count) { + if (this.result_count.hasOwnProperty(p)) { + this.elem.querySelector("td." + p).textContent = 0; + } + } + if (this.json_results_area) { + this.json_results_area.parentNode.removeChild(this.json_results_area); + } + this.meter.style.width = '0px'; + this.meter.textContent = '0%'; + this.manifest_status.style.display = "none"; + this.elem.querySelector(".jsonResults").style.display = "none"; + this.results_table.removeChild(this.results_table.tBodies[0]); + this.results_table.appendChild(document.createElement("tbody")); + }, + + on_manifest_wait: function() { + this.clear(); + this.instructions.style.display = "none"; + this.elem.style.display = "block"; + this.manifest_status.style.display = "inline"; + }, + + on_start: function() { + this.clear(); + this.instructions.style.display = "none"; + this.elem.style.display = "block"; + this.meter.classList.remove("stopped"); + this.meter.classList.add("progress-striped", "active"); + }, + + on_result: function(test, status, message, subtests) { + var row = document.createElement("tr"); + + var subtest_pass_count = subtests.reduce(function(prev, current) { + return (current.status === "PASS") ? prev + 1 : prev; + }, 0); + + var subtest_notrun_count = subtests.reduce(function(prev, current) { + return (current.status === "NOTRUN") ? prev +1 : prev; + }, 0); + + var subtests_count = subtests.length; + + var test_status; + if (subtest_pass_count === subtests_count && + (status == "OK" || status == "PASS")) { + test_status = "PASS"; + } else if (subtest_notrun_count == subtests_count) { + test_status = "NOTRUN"; + } else if (subtests_count > 0 && status === "OK") { + test_status = "FAIL"; + } else { + test_status = status; + } + + subtests.forEach(function(subtest) { + if (this.result_count.hasOwnProperty(subtest.status)) { + this.result_count[subtest.status] += 1; + } + }.bind(this)); + if (this.result_count.hasOwnProperty(status)) { + this.result_count[status] += 1; + } + + var name_node = row.appendChild(document.createElement("td")); + name_node.appendChild(this.test_name_node(test)); + + var status_node = row.appendChild(document.createElement("td")); + status_node.textContent = test_status; + status_node.className = test_status; + + var message_node = row.appendChild(document.createElement("td")); + message_node.textContent = message || ""; + + var subtests_node = row.appendChild(document.createElement("td")); + if (subtests_count) { + subtests_node.textContent = subtest_pass_count + "/" + subtests_count; + } else { + if (status == "PASS") { + subtests_node.textContent = "1/1"; + } else { + subtests_node.textContent = "0/1"; + } + } + + var status_arr = ["PASS", "FAIL", "ERROR", "TIMEOUT", "NOTRUN"]; + for (var i = 0; i < status_arr.length; i++) { + this.elem.querySelector("td." + status_arr[i]).textContent = this.result_count[status_arr[i]]; + } + + this.apply_display_filter_to_result_row(row, this.display_filter_state[test_status]); + this.results_table.tBodies[0].appendChild(row); + this.update_meter(this.runner.progress(), this.runner.results.count(), this.runner.test_count()); + }, + + on_done: function() { + this.meter.setAttribute("aria-valuenow", this.meter.getAttribute("aria-valuemax")); + this.meter.style.width = "100%"; + if (this.runner.stop_flag) { + this.meter.textContent = "Stopped"; + this.meter.classList.add("stopped"); + } else { + this.meter.textContent = "Done!"; + } + this.meter.classList.remove("progress-striped", "active"); + this.runner.test_div.textContent = ""; + //add the json serialization of the results + var a = this.elem.querySelector(".jsonResults"); + var json = this.runner.results.to_json(); + + if (document.getElementById("dumpit").checked) { + this.json_results_area = Array.prototype.slice.call(this.elem.querySelectorAll("textarea")); + for(var i = 0,t = this.json_results_area.length; i < t; i++){ + this.elem.removeChild(this.json_results_area[i]); + } + this.json_results_area = document.createElement("textarea"); + this.json_results_area.style.width = "100%"; + this.json_results_area.setAttribute("rows", "50"); + this.elem.appendChild(this.json_results_area); + this.json_results_area.textContent = json; + } + var blob = new Blob([json], { type: "application/json" }); + a.href = window.URL.createObjectURL(blob); + a.download = "runner-results.json"; + a.textContent = "Download JSON results"; + if (!a.getAttribute("download")) a.textContent += " (right-click and save as to download)"; + a.style.display = "inline"; + }, + + test_name_node: function(test) { + if (!test.hasOwnProperty("ref_url")) { + return this.link(test.url); + } else { + var wrapper = document.createElement("span"); + wrapper.appendChild(this.link(test.url)); + wrapper.appendChild(document.createTextNode(" " + test.ref_type + " ")); + wrapper.appendChild(this.link(test.ref_url)); + return wrapper; + } + }, + + link: function(href) { + var link = document.createElement("a"); + link.href = this.runner.server + href; + link.textContent = href; + return link; + }, + + update_meter: function(progress, count, total) { + this.meter.setAttribute("aria-valuenow", count); + this.meter.setAttribute("aria-valuemax", total); + this.meter.textContent = this.meter.style.width = (progress * 100).toFixed(1) + "%"; + }, + + apply_display_filter: function(test_status, display_state) { + this.display_filter_state[test_status] = display_state; + var result_cells = this.elem.querySelectorAll(".results > table tr td." + test_status); + for (var i = 0; i < result_cells.length; ++i) { + this.apply_display_filter_to_result_row(result_cells[i].parentNode, display_state) + } + }, + + apply_display_filter_to_result_row: function(result_row, display_state) { + result_row.style.display = display_state ? "" : "none"; + } +}; + +function ManualUI(elem, runner) { + this.elem = elem; + this.runner = runner; + this.pass_button = this.elem.querySelector("button.pass"); + this.fail_button = this.elem.querySelector("button.fail"); + this.ref_buttons = this.elem.querySelector(".reftestUI"); + this.ref_type = this.ref_buttons.querySelector(".refType"); + this.ref_warning = this.elem.querySelector(".reftestWarn"); + this.test_button = this.ref_buttons.querySelector("button.test"); + this.ref_button = this.ref_buttons.querySelector("button.ref"); + + this.hide(); + + this.runner.test_start_callbacks.push(this.on_test_start.bind(this)); + this.runner.test_pause_callbacks.push(this.hide.bind(this)); + this.runner.done_callbacks.push(this.on_done.bind(this)); + + this.pass_button.onclick = function() { + this.disable_buttons(); + this.runner.on_result("PASS", "", []); + }.bind(this); + + this.fail_button.onclick = function() { + this.disable_buttons(); + this.runner.on_result("FAIL", "", []); + }.bind(this); +} + +ManualUI.prototype = { + show: function() { + this.elem.style.display = "block"; + setTimeout(this.enable_buttons.bind(this), 200); + }, + + hide: function() { + this.elem.style.display = "none"; + }, + + show_ref: function() { + this.ref_buttons.style.display = "block"; + this.test_button.onclick = function() { + this.runner.load(this.runner.current_test.url); + }.bind(this); + this.ref_button.onclick = function() { + this.runner.load(this.runner.current_test.ref_url); + }.bind(this); + }, + + hide_ref: function() { + this.ref_buttons.style.display = "none"; + }, + + disable_buttons: function() { + this.pass_button.disabled = true; + this.fail_button.disabled = true; + }, + + enable_buttons: function() { + this.pass_button.disabled = false; + this.fail_button.disabled = false; + }, + + on_test_start: function(test) { + if (test.type == "manual" || test.type == "reftest") { + this.show(); + } else { + this.hide(); + } + if (test.type == "reftest") { + this.show_ref(); + this.ref_type.textContent = test.ref_type === "==" ? "equal" : "unequal"; + if (test.ref_length > 1) { + this.ref_warning.textContent = "WARNING: only presenting first of " + test.ref_length + " references"; + this.ref_warning.style.display = "inline"; + } else { + this.ref_warning.textContent = ""; + this.ref_warning.style.display = "none"; + } + } else { + this.hide_ref(); + } + }, + + on_done: function() { + this.hide(); + } +}; + +function TestControl(elem, runner) { + this.elem = elem; + this.path_input = this.elem.querySelector(".path"); + this.path_input.addEventListener("change", function() { + this.set_counts(); + }.bind(this), false); + this.use_regex_input = this.elem.querySelector("#use_regex"); + this.use_regex_input.addEventListener("change", function() { + this.set_counts(); + }.bind(this), false); + this.pause_button = this.elem.querySelector("button.togglePause"); + this.start_button = this.elem.querySelector("button.toggleStart"); + this.type_checkboxes = Array.prototype.slice.call( + this.elem.querySelectorAll("input[type=checkbox].test-type")); + this.type_checkboxes.forEach(function(elem) { + elem.addEventListener("change", function() { + this.set_counts(); + }.bind(this), + false); + elem.addEventListener("click", function() { + this.start_button.disabled = this.get_test_types().length < 1; + }.bind(this), + false); + }.bind(this)); + + this.timeout_input = this.elem.querySelector(".timeout_multiplier"); + this.render_checkbox = this.elem.querySelector(".render"); + this.testcount_area = this.elem.querySelector("#testcount"); + this.runner = runner; + this.runner.done_callbacks.push(this.on_done.bind(this)); + this.set_start(); + this.set_counts(); +} + +TestControl.prototype = { + set_start: function() { + this.start_button.disabled = this.get_test_types().length < 1; + this.pause_button.disabled = true; + this.start_button.textContent = "Start"; + this.path_input.disabled = false; + this.type_checkboxes.forEach(function(elem) { + elem.disabled = false; + }); + this.start_button.onclick = function() { + var path = this.get_path(); + var test_types = this.get_test_types(); + var settings = this.get_testharness_settings(); + var use_regex = this.get_use_regex(); + this.runner.start(path, test_types, settings, use_regex); + this.set_stop(); + this.set_pause(); + }.bind(this); + }, + + set_stop: function() { + clearTimeout(this.runner.timeout); + this.pause_button.disabled = false; + this.start_button.textContent = "Stop"; + this.path_input.disabled = true; + this.type_checkboxes.forEach(function(elem) { + elem.disabled = true; + }); + this.start_button.onclick = function() { + this.runner.stop_flag = true; + this.runner.done(); + }.bind(this); + }, + + set_pause: function() { + this.pause_button.textContent = "Pause"; + this.pause_button.onclick = function() { + this.runner.pause(); + this.set_resume(); + }.bind(this); + }, + + set_resume: function() { + this.pause_button.textContent = "Resume"; + this.pause_button.onclick = function() { + this.runner.unpause(); + this.set_pause(); + }.bind(this); + + }, + + set_counts: function() { + if (this.runner.manifest_loading) { + setTimeout(function() { + this.set_counts(); + }.bind(this), 1000); + return; + } + var path = this.get_path(); + var test_types = this.get_test_types(); + var use_regex = this.get_use_regex(); + var iterator = new ManifestIterator(this.runner.manifest, path, test_types, use_regex); + var count = iterator.count(); + this.testcount_area.textContent = count; + }, + + get_path: function() { + return this.path_input.value; + }, + + get_test_types: function() { + return this.type_checkboxes.filter(function(elem) { + return elem.checked; + }).map(function(elem) { + return elem.value; + }); + }, + + get_testharness_settings: function() { + return {timeout_multiplier: parseFloat(this.timeout_input.value), + output: this.render_checkbox.checked}; + }, + + get_use_regex: function() { + return this.use_regex_input.checked; + }, + + on_done: function() { + this.set_pause(); + this.set_start(); + } +}; + +function Results(runner) { + this.test_results = null; + this.runner = runner; + + this.runner.start_callbacks.push(this.on_start.bind(this)); +} + +Results.prototype = { + on_start: function() { + this.test_results = []; + }, + + set: function(test, status, message, subtests) { + this.test_results.push({"test":test, + "subtests":subtests, + "status":status, + "message":message}); + }, + + count: function() { + return this.test_results.length; + }, + + to_json: function() { + var data = { + "results": this.test_results.map(function(result) { + var rv = {"test":(result.test.hasOwnProperty("ref_url") ? + [result.test.url, result.test.ref_type, result.test.ref_url] : + result.test.url), + "subtests":result.subtests, + "status":result.status, + "message":result.message}; + return rv; + }) + }; + return JSON.stringify(data, null, 2); + } +}; + +function Runner(manifest_path) { + this.server = location.protocol + "//" + location.host; + this.manifest = new Manifest(manifest_path); + this.path = null; + this.test_types = null; + this.manifest_iterator = null; + + this.test_window = null; + this.test_div = document.getElementById('test_url'); + this.current_test = null; + this.timeout = null; + this.num_tests = null; + this.pause_flag = false; + this.stop_flag = false; + this.done_flag = false; + + this.manifest_wait_callbacks = []; + this.start_callbacks = []; + this.test_start_callbacks = []; + this.test_pause_callbacks = []; + this.result_callbacks = []; + this.done_callbacks = []; + + this.results = new Results(this); + + this.start_after_manifest_load = false; + this.manifest_loading = true; + this.manifest.load(this.manifest_loaded.bind(this)); +} + +Runner.prototype = { + test_timeout: 20000, //ms + + currentTest: function() { + return this.manifest[this.mTestCount]; + }, + + open_test_window: function() { + this.test_window = window.open("about:blank", 800, 600); + }, + + manifest_loaded: function() { + this.manifest_loading = false; + if (this.start_after_manifest_load) { + this.do_start(); + } + }, + + start: function(path, test_types, testharness_settings, use_regex) { + this.pause_flag = false; + this.stop_flag = false; + this.done_flag = false; + this.path = path; + this.use_regex = use_regex; + this.test_types = test_types; + window.testharness_properties = testharness_settings; + this.manifest_iterator = new ManifestIterator(this.manifest, this.path, this.test_types, this.use_regex); + this.num_tests = null; + + if (this.manifest.data === null) { + this.wait_for_manifest(); + } else { + this.do_start(); + } + }, + + wait_for_manifest: function() { + this.start_after_manifest_load = true; + this.manifest_wait_callbacks.forEach(function(callback) { + callback(); + }); + }, + + do_start: function() { + if (this.manifest_iterator.count() > 0) { + this.open_test_window(); + this.start_callbacks.forEach(function(callback) { + callback(); + }); + this.run_next_test(); + } else { + var tests = "tests"; + if (this.test_types.length < 3) { + tests = this.test_types.join(" tests or ") + " tests"; + } + var message = "No " + tests + " found in this path." + document.querySelector(".path").setCustomValidity(message); + this.done(); + } + }, + + pause: function() { + this.pause_flag = true; + this.test_pause_callbacks.forEach(function(callback) { + callback(this.current_test); + }.bind(this)); + }, + + unpause: function() { + this.pause_flag = false; + this.run_next_test(); + }, + + on_result: function(status, message, subtests) { + clearTimeout(this.timeout); + this.results.set(this.current_test, status, message, subtests); + this.result_callbacks.forEach(function(callback) { + callback(this.current_test, status, message, subtests); + }.bind(this)); + this.run_next_test(); + }, + + on_timeout: function() { + this.on_result("TIMEOUT", "", []); + }, + + done: function() { + this.done_flag = true; + if (this.test_window) { + this.test_window.close(); + } + this.done_callbacks.forEach(function(callback) { + callback(); + }); + }, + + run_next_test: function() { + if (this.pause_flag) { + return; + } + var next_test = this.manifest_iterator.next(); + if (next_test === null||this.done_flag) { + this.done(); + return; + } + + this.current_test = next_test; + + if (next_test.type === "testharness") { + this.timeout = setTimeout(this.on_timeout.bind(this), + this.test_timeout * window.testharness_properties.timeout_multiplier); + } + this.test_div.textContent = this.current_test.url; + this.load(this.current_test.url); + + this.test_start_callbacks.forEach(function(callback) { + callback(this.current_test); + }.bind(this)); + }, + + load: function(path) { + if (this.test_window.location === null) { + this.open_test_window(); + } + this.test_window.location.href = this.server + path; + }, + + progress: function() { + return this.results.count() / this.test_count(); + }, + + test_count: function() { + if (this.num_tests === null) { + this.num_tests = this.manifest_iterator.count(); + } + return this.num_tests; + } + +}; + + +function parseOptions() { + var options = { + test_types: ["testharness", "reftest", "manual"] + }; + + var optionstrings = location.search.substring(1).split("&"); + for (var i = 0, il = optionstrings.length; i < il; ++i) { + var opt = optionstrings[i]; + //TODO: fix this for complex-valued options + options[opt.substring(0, opt.indexOf("="))] = + opt.substring(opt.indexOf("=") + 1); + } + return options; +} + +function setup() { + var options = parseOptions(); + + if (options.path) { + document.getElementById('path').value = options.path; + } + + runner = new Runner("/MANIFEST.json", options); + var test_control = new TestControl(document.getElementById("testControl"), runner); + new ManualUI(document.getElementById("manualUI"), runner); + new VisualOutput(document.getElementById("output"), runner); + + if (options.autorun === "1") { + runner.start(test_control.get_path(), + test_control.get_test_types(), + test_control.get_testharness_settings(), + test_control.get_use_regex()); + return; + } +} + +window.completion_callback = function(tests, status) { + var harness_status_map = {0:"OK", 1:"ERROR", 2:"TIMEOUT", 3:"NOTRUN"}; + var subtest_status_map = {0:"PASS", 1:"FAIL", 2:"TIMEOUT", 3:"NOTRUN"}; + + // this ugly hack is because IE really insists on holding on to the objects it creates in + // other windows, and on losing track of them when the window gets closed + var subtest_results = JSON.parse(JSON.stringify( + tests.map(function (test) { + return {name: test.name, + status: subtest_status_map[test.status], + message: test.message}; + }) + )); + + runner.on_result(harness_status_map[status.status], + status.message, + subtest_results); +}; + +window.addEventListener("DOMContentLoaded", setup, false); +})(); |