summaryrefslogtreecommitdiffstats
path: root/testing/web-platform/tests/mixed-content/generic/tools
diff options
context:
space:
mode:
Diffstat (limited to 'testing/web-platform/tests/mixed-content/generic/tools')
-rw-r--r--testing/web-platform/tests/mixed-content/generic/tools/__init__.py0
-rwxr-xr-xtesting/web-platform/tests/mixed-content/generic/tools/clean.py35
-rw-r--r--testing/web-platform/tests/mixed-content/generic/tools/common_paths.py58
-rwxr-xr-xtesting/web-platform/tests/mixed-content/generic/tools/generate.py157
-rwxr-xr-xtesting/web-platform/tests/mixed-content/generic/tools/regenerate3
-rwxr-xr-xtesting/web-platform/tests/mixed-content/generic/tools/spec_validator.py159
6 files changed, 412 insertions, 0 deletions
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/__init__.py b/testing/web-platform/tests/mixed-content/generic/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/__init__.py
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/clean.py b/testing/web-platform/tests/mixed-content/generic/tools/clean.py
new file mode 100755
index 000000000..9416f0b5b
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/clean.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import os, json
+from common_paths import *
+import spec_validator
+
+def rmtree(top):
+ top = os.path.abspath(top)
+ assert top != os.path.expanduser("~")
+ assert len(top) > len(os.path.expanduser("~"))
+ assert "web-platform-tests" in top
+ assert "mixed-content" in top
+
+ for root, dirs, files in os.walk(top, topdown=False):
+ for name in files:
+ os.remove(os.path.join(root, name))
+ for name in dirs:
+ os.rmdir(os.path.join(root, name))
+
+ os.rmdir(top)
+
+def main():
+ spec_json = load_spec_json();
+ spec_validator.assert_valid_spec_json(spec_json)
+
+ for spec in spec_json['specification']:
+ generated_dir = os.path.join(spec_directory, spec["name"])
+ if (os.path.isdir(generated_dir)):
+ rmtree(generated_dir)
+
+ if (os.path.isfile(generated_spec_json_filename)):
+ os.remove(generated_spec_json_filename)
+
+if __name__ == '__main__':
+ main()
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py b/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py
new file mode 100644
index 000000000..5c2807d28
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/common_paths.py
@@ -0,0 +1,58 @@
+import os, sys, json, re
+
+script_directory = os.path.dirname(os.path.abspath(__file__))
+generic_directory = os.path.abspath(os.path.join(script_directory, '..'))
+
+template_directory = os.path.abspath(os.path.join(script_directory,
+ '..',
+ 'template'))
+spec_directory = os.path.abspath(os.path.join(script_directory, '..', '..'))
+test_root_directory = os.path.abspath(os.path.join(script_directory,
+ '..', '..', '..'))
+
+spec_filename = os.path.join(spec_directory, "spec.src.json")
+generated_spec_json_filename = os.path.join(spec_directory, "spec_json.js")
+
+selection_pattern = '%(opt_in_method)s/' + \
+ '%(origin)s/' + \
+ '%(subresource)s/' + \
+ '%(context_nesting)s/' + \
+ '%(redirection)s/'
+
+test_file_path_pattern = '%(spec_name)s/' + selection_pattern + \
+ '%(name)s.%(source_scheme)s.html'
+
+
+def get_template(basename):
+ with open(os.path.join(template_directory, basename), "r") as f:
+ return f.read()
+
+
+def write_file(filename, contents):
+ with open(filename, "w") as f:
+ f.write(contents)
+
+
+def read_nth_line(fp, line_number):
+ fp.seek(0)
+ for i, line in enumerate(fp):
+ if (i + 1) == line_number:
+ return line
+
+
+def load_spec_json(path_to_spec = None):
+ if path_to_spec is None:
+ path_to_spec = spec_filename
+
+ re_error_location = re.compile('line ([0-9]+) column ([0-9]+)')
+ with open(path_to_spec, "r") as f:
+ try:
+ return json.load(f)
+ except ValueError, ex:
+ print ex.message
+ match = re_error_location.search(ex.message)
+ if match:
+ line_number, column = int(match.group(1)), int(match.group(2))
+ print read_nth_line(f, line_number).rstrip()
+ print " " * (column - 1) + "^"
+ sys.exit(1)
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/generate.py b/testing/web-platform/tests/mixed-content/generic/tools/generate.py
new file mode 100755
index 000000000..6dcaebdc3
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/generate.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+
+import os, sys, json
+from common_paths import *
+import spec_validator
+import argparse
+
+
+def expand_pattern(expansion_pattern, test_expansion_schema):
+ expansion = {}
+ for artifact_key in expansion_pattern:
+ artifact_value = expansion_pattern[artifact_key]
+ if artifact_value == '*':
+ expansion[artifact_key] = test_expansion_schema[artifact_key]
+ elif isinstance(artifact_value, list):
+ expansion[artifact_key] = artifact_value
+ elif isinstance(artifact_value, dict):
+ # Flattened expansion.
+ expansion[artifact_key] = []
+ values_dict = expand_pattern(artifact_value,
+ test_expansion_schema[artifact_key])
+ for sub_key in values_dict.keys():
+ expansion[artifact_key] += values_dict[sub_key]
+ else:
+ expansion[artifact_key] = [artifact_value]
+
+ return expansion
+
+
+def permute_expansion(expansion, artifact_order, selection = {}, artifact_index = 0):
+ assert isinstance(artifact_order, list), "artifact_order should be a list"
+
+ if artifact_index >= len(artifact_order):
+ yield selection
+ return
+
+ artifact_key = artifact_order[artifact_index]
+
+ for artifact_value in expansion[artifact_key]:
+ selection[artifact_key] = artifact_value
+ for next_selection in permute_expansion(expansion,
+ artifact_order,
+ selection,
+ artifact_index + 1):
+ yield next_selection
+
+
+def generate_selection(selection, spec, test_html_template_basename):
+ selection['spec_name'] = spec['name']
+ selection['spec_title'] = spec['title']
+ selection['spec_description'] = spec['description']
+ selection['spec_specification_url'] = spec['specification_url']
+
+ test_filename = test_file_path_pattern % selection
+ test_headers_filename = test_filename + ".headers"
+ test_directory = os.path.dirname(test_filename)
+ full_path = os.path.join(spec_directory, test_directory)
+
+ test_html_template = get_template(test_html_template_basename)
+ test_js_template = get_template("test.js.template")
+ disclaimer_template = get_template('disclaimer.template')
+ test_description_template = get_template("test_description.template")
+
+ html_template_filename = os.path.join(template_directory,
+ test_html_template_basename)
+ generated_disclaimer = disclaimer_template \
+ % {'generating_script_filename': os.path.relpath(__file__,
+ test_root_directory),
+ 'html_template_filename': os.path.relpath(html_template_filename,
+ test_root_directory)}
+
+ selection['generated_disclaimer'] = generated_disclaimer.rstrip()
+ test_description_template = \
+ test_description_template.rstrip().replace("\n", "\n" + " " * 33)
+ selection['test_description'] = test_description_template % selection
+
+ # Adjust the template for the test invoking JS. Indent it to look nice.
+ indent = "\n" + " " * 6;
+ test_js_template = indent + test_js_template.replace("\n", indent);
+ selection['test_js'] = test_js_template % selection
+
+ # Directory for the test files.
+ try:
+ os.makedirs(full_path)
+ except:
+ pass
+
+ # TODO(kristijanburnik): Implement the opt-in-method here.
+ opt_in_method = selection['opt_in_method']
+ selection['meta_opt_in'] = ''
+ if opt_in_method == 'meta-csp':
+ selection['meta_opt_in'] = '\n <meta http-equiv="Content-Security-Policy" ' + \
+ 'content="block-all-mixed-content">'
+ elif opt_in_method == 'http-csp':
+ opt_in_headers = "Content-Security-Policy: block-all-mixed-content\n"
+ write_file(test_headers_filename, opt_in_headers)
+ elif opt_in_method == 'no-opt-in':
+ pass
+ else:
+ raise ValueError("Invalid opt_in_method %s" % opt_in_method)
+
+ # Write out the generated HTML file.
+ write_file(test_filename, test_html_template % selection)
+
+def generate_test_source_files(spec_json, target):
+ test_expansion_schema = spec_json['test_expansion_schema']
+ specification = spec_json['specification']
+
+ spec_json_js_template = get_template('spec_json.js.template')
+ write_file(generated_spec_json_filename,
+ spec_json_js_template % {'spec_json': json.dumps(spec_json)})
+
+ # Choose a debug/release template depending on the target.
+ html_template = "test.%s.html.template" % target
+
+ artifact_order = test_expansion_schema.keys() + ['name']
+
+ # Create list of excluded tests.
+ exclusion_dict = {}
+ for excluded_pattern in spec_json['excluded_tests']:
+ excluded_expansion = \
+ expand_pattern(excluded_pattern,
+ test_expansion_schema)
+ for excluded_selection in permute_expansion(excluded_expansion, artifact_order):
+ excluded_selection_path = selection_pattern % excluded_selection
+ exclusion_dict[excluded_selection_path] = True
+
+ for spec in specification:
+ for expansion_pattern in spec['test_expansion']:
+ expansion = expand_pattern(expansion_pattern,
+ test_expansion_schema)
+ for selection in permute_expansion(expansion, artifact_order):
+ selection_path = selection_pattern % selection
+ if not selection_path in exclusion_dict:
+ generate_selection(selection,
+ spec,
+ html_template)
+ else:
+ print 'Excluding selection:', selection_path
+
+
+def main(target, spec_filename):
+ spec_json = load_spec_json(spec_filename);
+ spec_validator.assert_valid_spec_json(spec_json)
+ generate_test_source_files(spec_json, target)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Test suite generator utility')
+ parser.add_argument('-t', '--target', type = str,
+ choices = ("release", "debug"), default = "release",
+ help = 'Sets the appropriate template for generating tests')
+ parser.add_argument('-s', '--spec', type = str, default = None,
+ help = 'Specify a file used for describing and generating the tests')
+ # TODO(kristijanburnik): Add option for the spec_json file.
+ args = parser.parse_args()
+ main(args.target, args.spec)
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/regenerate b/testing/web-platform/tests/mixed-content/generic/tools/regenerate
new file mode 100755
index 000000000..e6bd63519
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/regenerate
@@ -0,0 +1,3 @@
+#!/bin/bash
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+python $DIR/clean.py && python $DIR/generate.py
diff --git a/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py b/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py
new file mode 100755
index 000000000..a6acc1040
--- /dev/null
+++ b/testing/web-platform/tests/mixed-content/generic/tools/spec_validator.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+
+import json, sys
+from common_paths import *
+
+def assert_non_empty_string(obj, field):
+ assert field in obj, 'Missing field "%s"' % field
+ assert isinstance(obj[field], basestring), \
+ 'Field "%s" must be a string' % field
+ assert len(obj[field]) > 0, 'Field "%s" must not be empty' % field
+
+
+def assert_non_empty_list(obj, field):
+ assert isinstance(obj[field], list), \
+ '%s must be a list' % field
+ assert len(obj[field]) > 0, \
+ '%s list must not be empty' % field
+
+
+def assert_non_empty_dict(obj, field):
+ assert isinstance(obj[field], dict), \
+ '%s must be a dict' % field
+ assert len(obj[field]) > 0, \
+ '%s dict must not be empty' % field
+
+
+def assert_contains(obj, field):
+ assert field in obj, 'Must contain field "%s"' % field
+
+
+def assert_string_from(obj, field, items):
+ assert obj[field] in items, \
+ 'Field "%s" must be from: %s' % (field, str(items))
+
+
+def assert_string_or_list_items_from(obj, field, items):
+ if isinstance(obj[field], basestring):
+ assert_string_from(obj, field, items)
+ return
+
+ assert isinstance(obj[field], list), "%s must be a list!" % field
+ for allowed_value in obj[field]:
+ assert allowed_value != '*', "Wildcard is not supported for lists!"
+ assert allowed_value in items, \
+ 'Field "%s" must be from: %s' % (field, str(items))
+
+
+def assert_contains_only_fields(obj, expected_fields):
+ for expected_field in expected_fields:
+ assert_contains(obj, expected_field)
+
+ for actual_field in obj:
+ assert actual_field in expected_fields, \
+ 'Unexpected field "%s".' % actual_field
+
+
+def assert_value_unique_in(value, used_values):
+ assert value not in used_values, 'Duplicate value "%s"!' % str(value)
+ used_values[value] = True
+
+
+def assert_valid_artifact(exp_pattern, artifact_key, schema):
+ if isinstance(schema, list):
+ assert_string_or_list_items_from(exp_pattern, artifact_key,
+ ["*"] + schema)
+ return
+
+ for sub_artifact_key, sub_schema in schema.iteritems():
+ assert_valid_artifact(exp_pattern[artifact_key], sub_artifact_key,
+ sub_schema)
+
+def validate(spec_json, details):
+ """ Validates the json specification for generating tests. """
+
+ details['object'] = spec_json
+ assert_contains_only_fields(spec_json, ["specification",
+ "test_expansion_schema",
+ "excluded_tests"])
+ assert_non_empty_list(spec_json, "specification")
+ assert_non_empty_dict(spec_json, "test_expansion_schema")
+ assert_non_empty_list(spec_json, "excluded_tests")
+
+ specification = spec_json['specification']
+ test_expansion_schema = spec_json['test_expansion_schema']
+ excluded_tests = spec_json['excluded_tests']
+
+ valid_test_expansion_fields = ['name'] + test_expansion_schema.keys()
+
+ # Validate each single spec.
+ for spec in specification:
+ details['object'] = spec
+
+ # Validate required fields for a single spec.
+ assert_contains_only_fields(spec, ['name',
+ 'title',
+ 'description',
+ 'specification_url',
+ 'test_expansion'])
+ assert_non_empty_string(spec, 'name')
+ assert_non_empty_string(spec, 'title')
+ assert_non_empty_string(spec, 'description')
+ assert_non_empty_string(spec, 'specification_url')
+ assert_non_empty_list(spec, 'test_expansion')
+
+ # Validate spec's test expansion.
+ used_spec_names = {}
+
+ for spec_exp in spec['test_expansion']:
+ details['object'] = spec_exp
+ assert_non_empty_string(spec_exp, 'name')
+ # The name is unique in same expansion group.
+ assert_value_unique_in((spec_exp['expansion'], spec_exp['name']),
+ used_spec_names)
+ assert_contains_only_fields(spec_exp, valid_test_expansion_fields)
+
+ for artifact in test_expansion_schema:
+ details['test_expansion_field'] = artifact
+ assert_valid_artifact(spec_exp, artifact,
+ test_expansion_schema[artifact])
+ del details['test_expansion_field']
+
+ # Validate the test_expansion schema members.
+ details['object'] = test_expansion_schema
+ assert_contains_only_fields(test_expansion_schema, ['expansion',
+ 'source_scheme',
+ 'opt_in_method',
+ 'context_nesting',
+ 'redirection',
+ 'subresource',
+ 'origin',
+ 'expectation'])
+ # Validate excluded tests.
+ details['object'] = excluded_tests
+ for excluded_test_expansion in excluded_tests:
+ assert_contains_only_fields(excluded_test_expansion,
+ valid_test_expansion_fields)
+
+
+ del details['object']
+
+
+def assert_valid_spec_json(spec_json):
+ error_details = {}
+ try:
+ validate(spec_json, error_details)
+ except AssertionError, err:
+ print 'ERROR:', err.message
+ print json.dumps(error_details, indent=4)
+ sys.exit(1)
+
+
+def main():
+ spec_json = load_spec_json();
+ assert_valid_spec_json(spec_json)
+ print "Spec JSON is valid."
+
+
+if __name__ == '__main__':
+ main()