diff options
Diffstat (limited to 'testing/web-platform/tests/mixed-content/generic/tools')
6 files changed, 412 insertions, 0 deletions
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/__init__.py b/testing/web-platform/tests/mixed-content/generic/tools/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/__init__.py diff --git a/testing/web-platform/tests/mixed-content/generic/tools/clean.py b/testing/web-platform/tests/mixed-content/generic/tools/clean.py new file mode 100755 index 000000000..9416f0b5b --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/clean.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import os, json +from common_paths import * +import spec_validator + +def rmtree(top): + top = os.path.abspath(top) + assert top != os.path.expanduser("~") + assert len(top) > len(os.path.expanduser("~")) + assert "web-platform-tests" in top + assert "mixed-content" in top + + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + + os.rmdir(top) + +def main(): + spec_json = load_spec_json(); + spec_validator.assert_valid_spec_json(spec_json) + + for spec in spec_json['specification']: + generated_dir = os.path.join(spec_directory, spec["name"]) + if (os.path.isdir(generated_dir)): + rmtree(generated_dir) + + if (os.path.isfile(generated_spec_json_filename)): + os.remove(generated_spec_json_filename) + +if __name__ == '__main__': + main() diff --git a/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py b/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py new file mode 100644 index 000000000..5c2807d28 --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py @@ -0,0 +1,58 @@ +import os, sys, json, re + +script_directory = os.path.dirname(os.path.abspath(__file__)) +generic_directory = os.path.abspath(os.path.join(script_directory, '..')) + +template_directory = os.path.abspath(os.path.join(script_directory, + '..', + 'template')) +spec_directory = os.path.abspath(os.path.join(script_directory, '..', '..')) +test_root_directory = os.path.abspath(os.path.join(script_directory, + '..', '..', '..')) + +spec_filename = os.path.join(spec_directory, "spec.src.json") +generated_spec_json_filename = os.path.join(spec_directory, "spec_json.js") + +selection_pattern = '%(opt_in_method)s/' + \ + '%(origin)s/' + \ + '%(subresource)s/' + \ + '%(context_nesting)s/' + \ + '%(redirection)s/' + +test_file_path_pattern = '%(spec_name)s/' + selection_pattern + \ + '%(name)s.%(source_scheme)s.html' + + +def get_template(basename): + with open(os.path.join(template_directory, basename), "r") as f: + return f.read() + + +def write_file(filename, contents): + with open(filename, "w") as f: + f.write(contents) + + +def read_nth_line(fp, line_number): + fp.seek(0) + for i, line in enumerate(fp): + if (i + 1) == line_number: + return line + + +def load_spec_json(path_to_spec = None): + if path_to_spec is None: + path_to_spec = spec_filename + + re_error_location = re.compile('line ([0-9]+) column ([0-9]+)') + with open(path_to_spec, "r") as f: + try: + return json.load(f) + except ValueError, ex: + print ex.message + match = re_error_location.search(ex.message) + if match: + line_number, column = int(match.group(1)), int(match.group(2)) + print read_nth_line(f, line_number).rstrip() + print " " * (column - 1) + "^" + sys.exit(1) diff --git a/testing/web-platform/tests/mixed-content/generic/tools/generate.py b/testing/web-platform/tests/mixed-content/generic/tools/generate.py new file mode 100755 index 000000000..6dcaebdc3 --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/generate.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python + +import os, sys, json +from common_paths import * +import spec_validator +import argparse + + +def expand_pattern(expansion_pattern, test_expansion_schema): + expansion = {} + for artifact_key in expansion_pattern: + artifact_value = expansion_pattern[artifact_key] + if artifact_value == '*': + expansion[artifact_key] = test_expansion_schema[artifact_key] + elif isinstance(artifact_value, list): + expansion[artifact_key] = artifact_value + elif isinstance(artifact_value, dict): + # Flattened expansion. + expansion[artifact_key] = [] + values_dict = expand_pattern(artifact_value, + test_expansion_schema[artifact_key]) + for sub_key in values_dict.keys(): + expansion[artifact_key] += values_dict[sub_key] + else: + expansion[artifact_key] = [artifact_value] + + return expansion + + +def permute_expansion(expansion, artifact_order, selection = {}, artifact_index = 0): + assert isinstance(artifact_order, list), "artifact_order should be a list" + + if artifact_index >= len(artifact_order): + yield selection + return + + artifact_key = artifact_order[artifact_index] + + for artifact_value in expansion[artifact_key]: + selection[artifact_key] = artifact_value + for next_selection in permute_expansion(expansion, + artifact_order, + selection, + artifact_index + 1): + yield next_selection + + +def generate_selection(selection, spec, test_html_template_basename): + selection['spec_name'] = spec['name'] + selection['spec_title'] = spec['title'] + selection['spec_description'] = spec['description'] + selection['spec_specification_url'] = spec['specification_url'] + + test_filename = test_file_path_pattern % selection + test_headers_filename = test_filename + ".headers" + test_directory = os.path.dirname(test_filename) + full_path = os.path.join(spec_directory, test_directory) + + test_html_template = get_template(test_html_template_basename) + test_js_template = get_template("test.js.template") + disclaimer_template = get_template('disclaimer.template') + test_description_template = get_template("test_description.template") + + html_template_filename = os.path.join(template_directory, + test_html_template_basename) + generated_disclaimer = disclaimer_template \ + % {'generating_script_filename': os.path.relpath(__file__, + test_root_directory), + 'html_template_filename': os.path.relpath(html_template_filename, + test_root_directory)} + + selection['generated_disclaimer'] = generated_disclaimer.rstrip() + test_description_template = \ + test_description_template.rstrip().replace("\n", "\n" + " " * 33) + selection['test_description'] = test_description_template % selection + + # Adjust the template for the test invoking JS. Indent it to look nice. + indent = "\n" + " " * 6; + test_js_template = indent + test_js_template.replace("\n", indent); + selection['test_js'] = test_js_template % selection + + # Directory for the test files. + try: + os.makedirs(full_path) + except: + pass + + # TODO(kristijanburnik): Implement the opt-in-method here. + opt_in_method = selection['opt_in_method'] + selection['meta_opt_in'] = '' + if opt_in_method == 'meta-csp': + selection['meta_opt_in'] = '\n <meta http-equiv="Content-Security-Policy" ' + \ + 'content="block-all-mixed-content">' + elif opt_in_method == 'http-csp': + opt_in_headers = "Content-Security-Policy: block-all-mixed-content\n" + write_file(test_headers_filename, opt_in_headers) + elif opt_in_method == 'no-opt-in': + pass + else: + raise ValueError("Invalid opt_in_method %s" % opt_in_method) + + # Write out the generated HTML file. + write_file(test_filename, test_html_template % selection) + +def generate_test_source_files(spec_json, target): + test_expansion_schema = spec_json['test_expansion_schema'] + specification = spec_json['specification'] + + spec_json_js_template = get_template('spec_json.js.template') + write_file(generated_spec_json_filename, + spec_json_js_template % {'spec_json': json.dumps(spec_json)}) + + # Choose a debug/release template depending on the target. + html_template = "test.%s.html.template" % target + + artifact_order = test_expansion_schema.keys() + ['name'] + + # Create list of excluded tests. + exclusion_dict = {} + for excluded_pattern in spec_json['excluded_tests']: + excluded_expansion = \ + expand_pattern(excluded_pattern, + test_expansion_schema) + for excluded_selection in permute_expansion(excluded_expansion, artifact_order): + excluded_selection_path = selection_pattern % excluded_selection + exclusion_dict[excluded_selection_path] = True + + for spec in specification: + for expansion_pattern in spec['test_expansion']: + expansion = expand_pattern(expansion_pattern, + test_expansion_schema) + for selection in permute_expansion(expansion, artifact_order): + selection_path = selection_pattern % selection + if not selection_path in exclusion_dict: + generate_selection(selection, + spec, + html_template) + else: + print 'Excluding selection:', selection_path + + +def main(target, spec_filename): + spec_json = load_spec_json(spec_filename); + spec_validator.assert_valid_spec_json(spec_json) + generate_test_source_files(spec_json, target) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Test suite generator utility') + parser.add_argument('-t', '--target', type = str, + choices = ("release", "debug"), default = "release", + help = 'Sets the appropriate template for generating tests') + parser.add_argument('-s', '--spec', type = str, default = None, + help = 'Specify a file used for describing and generating the tests') + # TODO(kristijanburnik): Add option for the spec_json file. + args = parser.parse_args() + main(args.target, args.spec) diff --git a/testing/web-platform/tests/mixed-content/generic/tools/regenerate b/testing/web-platform/tests/mixed-content/generic/tools/regenerate new file mode 100755 index 000000000..e6bd63519 --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/regenerate @@ -0,0 +1,3 @@ +#!/bin/bash +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +python $DIR/clean.py && python $DIR/generate.py diff --git a/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py b/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py new file mode 100755 index 000000000..a6acc1040 --- /dev/null +++ b/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +import json, sys +from common_paths import * + +def assert_non_empty_string(obj, field): + assert field in obj, 'Missing field "%s"' % field + assert isinstance(obj[field], basestring), \ + 'Field "%s" must be a string' % field + assert len(obj[field]) > 0, 'Field "%s" must not be empty' % field + + +def assert_non_empty_list(obj, field): + assert isinstance(obj[field], list), \ + '%s must be a list' % field + assert len(obj[field]) > 0, \ + '%s list must not be empty' % field + + +def assert_non_empty_dict(obj, field): + assert isinstance(obj[field], dict), \ + '%s must be a dict' % field + assert len(obj[field]) > 0, \ + '%s dict must not be empty' % field + + +def assert_contains(obj, field): + assert field in obj, 'Must contain field "%s"' % field + + +def assert_string_from(obj, field, items): + assert obj[field] in items, \ + 'Field "%s" must be from: %s' % (field, str(items)) + + +def assert_string_or_list_items_from(obj, field, items): + if isinstance(obj[field], basestring): + assert_string_from(obj, field, items) + return + + assert isinstance(obj[field], list), "%s must be a list!" % field + for allowed_value in obj[field]: + assert allowed_value != '*', "Wildcard is not supported for lists!" + assert allowed_value in items, \ + 'Field "%s" must be from: %s' % (field, str(items)) + + +def assert_contains_only_fields(obj, expected_fields): + for expected_field in expected_fields: + assert_contains(obj, expected_field) + + for actual_field in obj: + assert actual_field in expected_fields, \ + 'Unexpected field "%s".' % actual_field + + +def assert_value_unique_in(value, used_values): + assert value not in used_values, 'Duplicate value "%s"!' % str(value) + used_values[value] = True + + +def assert_valid_artifact(exp_pattern, artifact_key, schema): + if isinstance(schema, list): + assert_string_or_list_items_from(exp_pattern, artifact_key, + ["*"] + schema) + return + + for sub_artifact_key, sub_schema in schema.iteritems(): + assert_valid_artifact(exp_pattern[artifact_key], sub_artifact_key, + sub_schema) + +def validate(spec_json, details): + """ Validates the json specification for generating tests. """ + + details['object'] = spec_json + assert_contains_only_fields(spec_json, ["specification", + "test_expansion_schema", + "excluded_tests"]) + assert_non_empty_list(spec_json, "specification") + assert_non_empty_dict(spec_json, "test_expansion_schema") + assert_non_empty_list(spec_json, "excluded_tests") + + specification = spec_json['specification'] + test_expansion_schema = spec_json['test_expansion_schema'] + excluded_tests = spec_json['excluded_tests'] + + valid_test_expansion_fields = ['name'] + test_expansion_schema.keys() + + # Validate each single spec. + for spec in specification: + details['object'] = spec + + # Validate required fields for a single spec. + assert_contains_only_fields(spec, ['name', + 'title', + 'description', + 'specification_url', + 'test_expansion']) + assert_non_empty_string(spec, 'name') + assert_non_empty_string(spec, 'title') + assert_non_empty_string(spec, 'description') + assert_non_empty_string(spec, 'specification_url') + assert_non_empty_list(spec, 'test_expansion') + + # Validate spec's test expansion. + used_spec_names = {} + + for spec_exp in spec['test_expansion']: + details['object'] = spec_exp + assert_non_empty_string(spec_exp, 'name') + # The name is unique in same expansion group. + assert_value_unique_in((spec_exp['expansion'], spec_exp['name']), + used_spec_names) + assert_contains_only_fields(spec_exp, valid_test_expansion_fields) + + for artifact in test_expansion_schema: + details['test_expansion_field'] = artifact + assert_valid_artifact(spec_exp, artifact, + test_expansion_schema[artifact]) + del details['test_expansion_field'] + + # Validate the test_expansion schema members. + details['object'] = test_expansion_schema + assert_contains_only_fields(test_expansion_schema, ['expansion', + 'source_scheme', + 'opt_in_method', + 'context_nesting', + 'redirection', + 'subresource', + 'origin', + 'expectation']) + # Validate excluded tests. + details['object'] = excluded_tests + for excluded_test_expansion in excluded_tests: + assert_contains_only_fields(excluded_test_expansion, + valid_test_expansion_fields) + + + del details['object'] + + +def assert_valid_spec_json(spec_json): + error_details = {} + try: + validate(spec_json, error_details) + except AssertionError, err: + print 'ERROR:', err.message + print json.dumps(error_details, indent=4) + sys.exit(1) + + +def main(): + spec_json = load_spec_json(); + assert_valid_spec_json(spec_json) + print "Spec JSON is valid." + + +if __name__ == '__main__': + main() |