diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/wptmanifest')
13 files changed, 2539 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/__init__.py b/testing/web-platform/harness/wptrunner/wptmanifest/__init__.py new file mode 100644 index 000000000..365448467 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/__init__.py @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from serializer import serialize +from parser import parse +from backends.static import compile as compile_static +from backends.conditional import compile as compile_condition diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/backends/__init__.py b/testing/web-platform/harness/wptrunner/wptmanifest/backends/__init__.py new file mode 100644 index 000000000..c580d191c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/backends/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/backends/conditional.py b/testing/web-platform/harness/wptrunner/wptmanifest/backends/conditional.py new file mode 100644 index 000000000..56999b993 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/backends/conditional.py @@ -0,0 +1,334 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import operator + +from ..node import NodeVisitor, DataNode, ConditionalNode, KeyValueNode, ListNode, ValueNode +from ..parser import parse + + +class ConditionalValue(object): + def __init__(self, node, condition_func): + self.node = node + self.condition_func = condition_func + if isinstance(node, ConditionalNode): + assert len(node.children) == 2 + self.condition_node = self.node.children[0] + self.value_node = self.node.children[1] + else: + assert isinstance(node, (ValueNode, ListNode)) + self.condition_node = None + self.value_node = self.node + + @property + def value(self): + if isinstance(self.value_node, ValueNode): + return self.value_node.data + else: + return [item.data for item in self.value_node.children] + + @value.setter + def value(self, value): + self.value_node.data = value + + def __call__(self, run_info): + return self.condition_func(run_info) + + def set_value(self, value): + self.value = value + + def remove(self): + if len(self.node.parent.children) == 1: + self.node.parent.remove() + self.node.remove() + + +class Compiler(NodeVisitor): + def compile(self, tree, data_cls_getter=None, **kwargs): + """Compile a raw AST into a form where conditional expressions + are represented by ConditionalValue objects that can be evaluated + at runtime. + + tree - The root node of the wptmanifest AST to compile + + data_cls_getter - A function taking two parameters; the previous + output node and the current ast node and returning + the class of the output node to use for the current + ast node + """ + if data_cls_getter is None: + self.data_cls_getter = lambda x, y: ManifestItem + else: + self.data_cls_getter = data_cls_getter + + self.tree = tree + self.output_node = self._initial_output_node(tree, **kwargs) + self.visit(tree) + assert self.output_node is not None + return self.output_node + + def compile_condition(self, condition): + """Compile a ConditionalNode into a ConditionalValue. + + condition: A ConditionalNode""" + data_node = DataNode() + key_value_node = KeyValueNode() + key_value_node.append(condition.copy()) + data_node.append(key_value_node) + manifest_item = self.compile(data_node) + return manifest_item._data[None][0] + + def _initial_output_node(self, node, **kwargs): + return self.data_cls_getter(None, None)(node, **kwargs) + + def visit_DataNode(self, node): + if node != self.tree: + output_parent = self.output_node + self.output_node = self.data_cls_getter(self.output_node, node)(node) + else: + output_parent = None + + assert self.output_node is not None + + for child in node.children: + self.visit(child) + + if output_parent is not None: + # Append to the parent *after* processing all the node data + output_parent.append(self.output_node) + self.output_node = self.output_node.parent + + assert self.output_node is not None + + def visit_KeyValueNode(self, node): + key_values = [] + for child in node.children: + condition, value = self.visit(child) + key_values.append(ConditionalValue(child, condition)) + + self.output_node._add_key_value(node, key_values) + + def visit_ListNode(self, node): + return (lambda x:True, [self.visit(child) for child in node.children]) + + def visit_ValueNode(self, node): + return (lambda x: True, node.data) + + def visit_AtomNode(self, node): + return (lambda x: True, node.data) + + def visit_ConditionalNode(self, node): + return self.visit(node.children[0]), self.visit(node.children[1]) + + def visit_StringNode(self, node): + indexes = [self.visit(child) for child in node.children] + + def value(x): + rv = node.data + for index in indexes: + rv = rv[index(x)] + return rv + return value + + def visit_NumberNode(self, node): + if "." in node.data: + return lambda x: float(node.data) + else: + return lambda x: int(node.data) + + def visit_VariableNode(self, node): + indexes = [self.visit(child) for child in node.children] + + def value(x): + data = x[node.data] + for index in indexes: + data = data[index(x)] + return data + return value + + def visit_IndexNode(self, node): + assert len(node.children) == 1 + return self.visit(node.children[0]) + + def visit_UnaryExpressionNode(self, node): + assert len(node.children) == 2 + operator = self.visit(node.children[0]) + operand = self.visit(node.children[1]) + + return lambda x: operator(operand(x)) + + def visit_BinaryExpressionNode(self, node): + assert len(node.children) == 3 + operator = self.visit(node.children[0]) + operand_0 = self.visit(node.children[1]) + operand_1 = self.visit(node.children[2]) + + assert operand_0 is not None + assert operand_1 is not None + + return lambda x: operator(operand_0(x), operand_1(x)) + + def visit_UnaryOperatorNode(self, node): + return {"not": operator.not_}[node.data] + + def visit_BinaryOperatorNode(self, node): + return {"and": operator.and_, + "or": operator.or_, + "==": operator.eq, + "!=": operator.ne}[node.data] + + +class ManifestItem(object): + def __init__(self, node=None, **kwargs): + self.node = node + self.parent = None + self.children = [] + self._data = {} + + def __repr__(self): + return "<ManifestItem %s>" % (self.node.data) + + def __str__(self): + rv = [repr(self)] + for item in self.children: + rv.extend(" %s" % line for line in str(item).split("\n")) + return "\n".join(rv) + + def __contains__(self, key): + return key in self._data + + @property + def is_empty(self): + if self._data: + return False + return all(child.is_empty for child in self.children) + + @property + def root(self): + node = self + while node.parent is not None: + node = node.parent + return node + + @property + def name(self): + return self.node.data + + def has_key(self, key): + for node in [self, self.root]: + if key in node._data: + return True + return False + + def get(self, key, run_info=None): + if run_info is None: + run_info = {} + + for node in [self, self.root]: + if key in node._data: + for cond_value in node._data[key]: + try: + matches = cond_value(run_info) + except KeyError: + matches = False + if matches: + return cond_value.value + raise KeyError + + def set(self, key, value, condition=None): + # First try to update the existing value + if key in self._data: + cond_values = self._data[key] + for cond_value in cond_values: + if cond_value.condition_node == condition: + cond_value.value = value + return + # If there isn't a conditional match reuse the existing KeyValueNode as the + # parent + node = None + for child in self.node.children: + if child.data == key: + node = child + break + assert node is not None + + else: + node = KeyValueNode(key) + self.node.append(node) + + value_node = ValueNode(value) + if condition is not None: + conditional_node = ConditionalNode() + conditional_node.append(condition) + conditional_node.append(value_node) + node.append(conditional_node) + cond_value = Compiler().compile_condition(conditional_node) + else: + node.append(value_node) + cond_value = ConditionalValue(value_node, lambda x: True) + + # Update the cache of child values. This is pretty annoying and maybe + # it should just work directly on the tree + if key not in self._data: + self._data[key] = [] + if self._data[key] and self._data[key][-1].condition_node is None: + self._data[key].insert(len(self._data[key]) - 1, cond_value) + else: + self._data[key].append(cond_value) + + def _add_key_value(self, node, values): + """Called during construction to set a key-value node""" + self._data[node.data] = values + + def append(self, child): + self.children.append(child) + child.parent = self + if child.node.parent != self.node: + self.node.append(child.node) + return child + + def remove(self): + if self.parent: + self.parent._remove_child(self) + + def _remove_child(self, child): + self.children.remove(child) + child.parent = None + + def iterchildren(self, name=None): + for item in self.children: + if item.name == name or name is None: + yield item + + def _flatten(self): + rv = {} + for node in [self, self.root]: + for name, value in node._data.iteritems(): + if name not in rv: + rv[name] = value + return rv + + def iteritems(self): + for item in self._flatten().iteritems(): + yield item + + def iterkeys(self): + for item in self._flatten().iterkeys(): + yield item + + def remove_value(self, key, value): + self._data[key].remove(value) + if not self._data[key]: + del self._data[key] + value.remove() + + +def compile_ast(ast, data_cls_getter=None, **kwargs): + return Compiler().compile(ast, data_cls_getter=data_cls_getter, **kwargs) + + +def compile(stream, data_cls_getter=None, **kwargs): + return compile_ast(parse(stream), + data_cls_getter=data_cls_getter, + **kwargs) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/backends/static.py b/testing/web-platform/harness/wptrunner/wptmanifest/backends/static.py new file mode 100644 index 000000000..7221fce72 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/backends/static.py @@ -0,0 +1,224 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import operator + +from ..node import NodeVisitor +from ..parser import parse + + +class Compiler(NodeVisitor): + """Compiler backend that evaluates conditional expressions + to give static output""" + + def compile(self, tree, expr_data, data_cls_getter=None, **kwargs): + """Compile a raw AST into a form with conditional expressions + evaluated. + + tree - The root node of the wptmanifest AST to compile + + expr_data - A dictionary of key / value pairs to use when + evaluating conditional expressions + + data_cls_getter - A function taking two parameters; the previous + output node and the current ast node and returning + the class of the output node to use for the current + ast node + """ + + self._kwargs = kwargs + self.expr_data = expr_data + + if data_cls_getter is None: + self.data_cls_getter = lambda x, y: ManifestItem + else: + self.data_cls_getter = data_cls_getter + + self.output_node = None + self.visit(tree) + return self.output_node + + def visit_DataNode(self, node): + output_parent = self.output_node + if self.output_node is None: + assert node.parent is None + self.output_node = self.data_cls_getter(None, None)(None, **self._kwargs) + else: + self.output_node = self.data_cls_getter(self.output_node, node)(node.data) + + for child in node.children: + self.visit(child) + + if output_parent is not None: + output_parent.append(self.output_node) + self.output_node = self.output_node.parent + + def visit_KeyValueNode(self, node): + key_name = node.data + key_value = None + for child in node.children: + value = self.visit(child) + if value is not None: + key_value = value + break + if key_value is not None: + self.output_node.set(key_name, key_value) + + def visit_ValueNode(self, node): + return node.data + + def visit_AtomNode(self, node): + return node.data + + def visit_ListNode(self, node): + return [self.visit(child) for child in node.children] + + def visit_ConditionalNode(self, node): + assert len(node.children) == 2 + if self.visit(node.children[0]): + return self.visit(node.children[1]) + + def visit_StringNode(self, node): + value = node.data + for child in node.children: + value = self.visit(child)(value) + return value + + def visit_NumberNode(self, node): + if "." in node.data: + return float(node.data) + else: + return int(node.data) + + def visit_VariableNode(self, node): + value = self.expr_data[node.data] + for child in node.children: + value = self.visit(child)(value) + return value + + def visit_IndexNode(self, node): + assert len(node.children) == 1 + index = self.visit(node.children[0]) + return lambda x: x[index] + + def visit_UnaryExpressionNode(self, node): + assert len(node.children) == 2 + operator = self.visit(node.children[0]) + operand = self.visit(node.children[1]) + + return operator(operand) + + def visit_BinaryExpressionNode(self, node): + assert len(node.children) == 3 + operator = self.visit(node.children[0]) + operand_0 = self.visit(node.children[1]) + operand_1 = self.visit(node.children[2]) + + return operator(operand_0, operand_1) + + def visit_UnaryOperatorNode(self, node): + return {"not": operator.not_}[node.data] + + def visit_BinaryOperatorNode(self, node): + return {"and": operator.and_, + "or": operator.or_, + "==": operator.eq, + "!=": operator.ne}[node.data] + + +class ManifestItem(object): + def __init__(self, name, **kwargs): + self.parent = None + self.name = name + self.children = [] + self._data = {} + + def __repr__(self): + return "<ManifestItem %s>" % (self.name) + + def __str__(self): + rv = [repr(self)] + for item in self.children: + rv.extend(" %s" % line for line in str(item).split("\n")) + return "\n".join(rv) + + @property + def is_empty(self): + if self._data: + return False + return all(child.is_empty for child in self.children) + + @property + def root(self): + node = self + while node.parent is not None: + node = node.parent + return node + + def has_key(self, key): + for node in [self, self.root]: + if key in node._data: + return True + return False + + def get(self, key): + for node in [self, self.root]: + if key in node._data: + return node._data[key] + raise KeyError + + def set(self, name, value): + self._data[name] = value + + def remove(self): + if self.parent: + self.parent._remove_child(self) + + def _remove_child(self, child): + self.children.remove(child) + child.parent = None + + def iterchildren(self, name=None): + for item in self.children: + if item.name == name or name is None: + yield item + + def _flatten(self): + rv = {} + for node in [self, self.root]: + for name, value in node._data.iteritems(): + if name not in rv: + rv[name] = value + return rv + + def iteritems(self): + for item in self._flatten().iteritems(): + yield item + + def iterkeys(self): + for item in self._flatten().iterkeys(): + yield item + + def itervalues(self): + for item in self._flatten().itervalues(): + yield item + + def append(self, child): + child.parent = self + self.children.append(child) + return child + + +def compile_ast(ast, expr_data, data_cls_getter=None, **kwargs): + return Compiler().compile(ast, + expr_data, + data_cls_getter=data_cls_getter, + **kwargs) + + +def compile(stream, expr_data, data_cls_getter=None, **kwargs): + return compile_ast(parse(stream), + expr_data, + data_cls_getter=data_cls_getter, + **kwargs) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/node.py b/testing/web-platform/harness/wptrunner/wptmanifest/node.py new file mode 100644 index 000000000..e260eeaf7 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/node.py @@ -0,0 +1,161 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +class NodeVisitor(object): + def visit(self, node): + # This is ugly as hell, but we don't have multimethods and + # they aren't trivial to fake without access to the class + # object from the class body + func = getattr(self, "visit_%s" % (node.__class__.__name__)) + return func(node) + + +class Node(object): + def __init__(self, data=None): + self.data = data + self.parent = None + self.children = [] + + def append(self, other): + other.parent = self + self.children.append(other) + + def remove(self): + self.parent.children.remove(self) + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.data) + + def __str__(self): + rv = [repr(self)] + for item in self.children: + rv.extend(" %s" % line for line in str(item).split("\n")) + return "\n".join(rv) + + def __eq__(self, other): + if not (self.__class__ == other.__class__ and + self.data == other.data and + len(self.children) == len(other.children)): + return False + for child, other_child in zip(self.children, other.children): + if not child == other_child: + return False + return True + + def copy(self): + new = self.__class__(self.data) + for item in self.children: + new.append(item.copy()) + return new + + +class DataNode(Node): + def append(self, other): + # Append that retains the invariant that child data nodes + # come after child nodes of other types + other.parent = self + if isinstance(other, DataNode): + self.children.append(other) + else: + index = len(self.children) + while index > 0 and isinstance(self.children[index - 1], DataNode): + index -= 1 + for i in xrange(index): + assert other.data != self.children[i].data + self.children.insert(index, other) + + +class KeyValueNode(Node): + def append(self, other): + # Append that retains the invariant that conditional nodes + # come before unconditional nodes + other.parent = self + if isinstance(other, ValueNode): + if self.children: + assert not isinstance(self.children[-1], ValueNode) + self.children.append(other) + else: + if self.children and isinstance(self.children[-1], ValueNode): + self.children.insert(len(self.children) - 1, other) + else: + self.children.append(other) + + +class ListNode(Node): + def append(self, other): + other.parent = self + self.children.append(other) + + +class ValueNode(Node): + def append(self, other): + raise TypeError + + +class AtomNode(ValueNode): + pass + + +class ConditionalNode(Node): + pass + + +class UnaryExpressionNode(Node): + def __init__(self, operator, operand): + Node.__init__(self) + self.append(operator) + self.append(operand) + + def append(self, other): + Node.append(self, other) + assert len(self.children) <= 2 + + def copy(self): + new = self.__class__(self.children[0].copy(), + self.children[1].copy()) + return new + + +class BinaryExpressionNode(Node): + def __init__(self, operator, operand_0, operand_1): + Node.__init__(self) + self.append(operator) + self.append(operand_0) + self.append(operand_1) + + def append(self, other): + Node.append(self, other) + assert len(self.children) <= 3 + + def copy(self): + new = self.__class__(self.children[0].copy(), + self.children[1].copy(), + self.children[2].copy()) + return new + + +class UnaryOperatorNode(Node): + def append(self, other): + raise TypeError + + +class BinaryOperatorNode(Node): + def append(self, other): + raise TypeError + + +class IndexNode(Node): + pass + + +class VariableNode(Node): + pass + + +class StringNode(Node): + pass + + +class NumberNode(ValueNode): + pass diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/parser.py b/testing/web-platform/harness/wptrunner/wptmanifest/parser.py new file mode 100644 index 000000000..eeac66d31 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/parser.py @@ -0,0 +1,744 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#default_value:foo +#include: other.manifest +# +#[test_name.js] +# expected: ERROR +# +# [subtest 1] +# expected: +# os == win: FAIL #This is a comment +# PASS +# + +# TODO: keep comments in the tree + +import types +from cStringIO import StringIO + +from node import * + + +class ParseError(Exception): + def __init__(self, filename, line, detail): + self.line = line + self.filename = filename + self.detail = detail + self.message = "%s: %s line %s" % (self.detail, self.filename, self.line) + Exception.__init__(self, self.message) + +eol = object +group_start = object +group_end = object +digits = "0123456789" +open_parens = "[(" +close_parens = "])" +parens = open_parens + close_parens +operator_chars = "=!" + +unary_operators = ["not"] +binary_operators = ["==", "!=", "and", "or"] + +operators = ["==", "!=", "not", "and", "or"] + +atoms = {"True": True, + "False": False, + "Reset": object()} + +def decode(byte_str): + return byte_str.decode("utf8") + + +def precedence(operator_node): + return len(operators) - operators.index(operator_node.data) + + +class TokenTypes(object): + def __init__(self): + for type in ["group_start", "group_end", "paren", "list_start", "list_end", "separator", "ident", "string", "number", "atom", "eof"]: + setattr(self, type, type) + +token_types = TokenTypes() + + +class Tokenizer(object): + def __init__(self): + self.reset() + + def reset(self): + self.indent_levels = [0] + self.state = self.line_start_state + self.next_state = self.data_line_state + self.line_number = 0 + + def tokenize(self, stream): + self.reset() + if type(stream) in types.StringTypes: + stream = StringIO(stream) + if not hasattr(stream, "name"): + self.filename = "" + else: + self.filename = stream.name + + self.next_line_state = self.line_start_state + for i, line in enumerate(stream): + self.state = self.next_line_state + assert self.state is not None + states = [] + self.next_line_state = None + self.line_number = i + 1 + self.index = 0 + self.line = line.rstrip() + while self.state != self.eol_state: + states.append(self.state) + tokens = self.state() + if tokens: + for token in tokens: + yield token + self.state() + while True: + yield (token_types.eof, None) + + def char(self): + if self.index == len(self.line): + return eol + return self.line[self.index] + + def consume(self): + if self.index < len(self.line): + self.index += 1 + + def peek(self, length): + return self.line[self.index:self.index + length] + + def skip_whitespace(self): + while self.char() == " ": + self.consume() + + def eol_state(self): + if self.next_line_state is None: + self.next_line_state = self.line_start_state + + def line_start_state(self): + self.skip_whitespace() + if self.char() == eol: + self.state = self.eol_state + return + if self.index > self.indent_levels[-1]: + self.indent_levels.append(self.index) + yield (token_types.group_start, None) + else: + while self.index < self.indent_levels[-1]: + self.indent_levels.pop() + yield (token_types.group_end, None) + # This is terrible; if we were parsing an expression + # then the next_state will be expr_or_value but when we deindent + # it must always be a heading or key next so we go back to data_line_state + self.next_state = self.data_line_state + if self.index != self.indent_levels[-1]: + raise ParseError(self.filename, self.line_number, "Unexpected indent") + + self.state = self.next_state + + def data_line_state(self): + if self.char() == "[": + yield (token_types.paren, self.char()) + self.consume() + self.state = self.heading_state + else: + self.state = self.key_state + + def heading_state(self): + rv = "" + while True: + c = self.char() + if c == "\\": + rv += self.consume_escape() + elif c == "]": + break + elif c == eol: + raise ParseError(self.filename, self.line_number, "EOL in heading") + else: + rv += c + self.consume() + + yield (token_types.string, decode(rv)) + yield (token_types.paren, "]") + self.consume() + self.state = self.line_end_state + self.next_state = self.data_line_state + + def key_state(self): + rv = "" + while True: + c = self.char() + if c == " ": + self.skip_whitespace() + if self.char() != ":": + raise ParseError(self.filename, self.line_number, "Space in key name") + break + elif c == ":": + break + elif c == eol: + raise ParseError(self.filename, self.line_number, "EOL in key name (missing ':'?)") + elif c == "\\": + rv += self.consume_escape() + else: + rv += c + self.consume() + yield (token_types.string, decode(rv)) + yield (token_types.separator, ":") + self.consume() + self.state = self.after_key_state + + def after_key_state(self): + self.skip_whitespace() + c = self.char() + if c == "#": + self.next_state = self.expr_or_value_state + self.state = self.comment_state + elif c == eol: + self.next_state = self.expr_or_value_state + self.state = self.eol_state + elif c == "[": + self.state = self.list_start_state + else: + self.state = self.value_state + + def list_start_state(self): + yield (token_types.list_start, "[") + self.consume() + self.state = self.list_value_start_state + + def list_value_start_state(self): + self.skip_whitespace() + if self.char() == "]": + self.state = self.list_end_state + elif self.char() in ("'", '"'): + quote_char = self.char() + self.consume() + yield (token_types.string, self.consume_string(quote_char)) + self.skip_whitespace() + if self.char() == "]": + self.state = self.list_end_state + elif self.char() != ",": + raise ParseError(self.filename, self.line_number, "Junk after quoted string") + self.consume() + elif self.char() == "#": + self.state = self.comment_state + self.next_line_state = self.list_value_start_state + elif self.char() == eol: + self.next_line_state = self.list_value_start_state + self.state = self.eol_state + elif self.char() == ",": + raise ParseError(self.filename, self.line_number, "List item started with separator") + elif self.char() == "@": + self.state = self.list_value_atom_state + else: + self.state = self.list_value_state + + def list_value_state(self): + rv = "" + spaces = 0 + while True: + c = self.char() + if c == "\\": + escape = self.consume_escape() + rv += escape + elif c == eol: + raise ParseError(self.filename, self.line_number, "EOL in list value") + elif c == "#": + raise ParseError(self.filename, self.line_number, "EOL in list value (comment)") + elif c == ",": + self.state = self.list_value_start_state + self.consume() + break + elif c == " ": + spaces += 1 + self.consume() + elif c == "]": + self.state = self.list_end_state + self.consume() + break + else: + rv += " " * spaces + spaces = 0 + rv += c + self.consume() + + if rv: + yield (token_types.string, decode(rv)) + + def list_value_atom_state(self): + self.consume() + for _, value in self.list_value_state(): + yield token_types.atom, value + + def list_end_state(self): + self.consume() + yield (token_types.list_end, "]") + self.state = self.line_end_state + + def value_state(self): + self.skip_whitespace() + if self.char() in ("'", '"'): + quote_char = self.char() + self.consume() + yield (token_types.string, self.consume_string(quote_char)) + if self.char() == "#": + self.state = self.comment_state + else: + self.state = self.line_end_state + elif self.char() == "@": + self.consume() + for _, value in self.value_inner_state(): + yield token_types.atom, value + else: + self.state = self.value_inner_state + + def value_inner_state(self): + rv = "" + spaces = 0 + while True: + c = self.char() + if c == "\\": + rv += self.consume_escape() + elif c == "#": + self.state = self.comment_state + break + elif c == " ": + # prevent whitespace before comments from being included in the value + spaces += 1 + self.consume() + elif c == eol: + self.state = self.line_end_state + break + else: + rv += " " * spaces + spaces = 0 + rv += c + self.consume() + yield (token_types.string, decode(rv)) + + def comment_state(self): + while self.char() is not eol: + self.consume() + self.state = self.eol_state + + def line_end_state(self): + self.skip_whitespace() + c = self.char() + if c == "#": + self.state = self.comment_state + elif c == eol: + self.state = self.eol_state + else: + raise ParseError(self.filename, self.line_number, "Junk before EOL %s" % c) + + def consume_string(self, quote_char): + rv = "" + while True: + c = self.char() + if c == "\\": + rv += self.consume_escape() + elif c == quote_char: + self.consume() + break + elif c == eol: + raise ParseError(self.filename, self.line_number, "EOL in quoted string") + else: + rv += c + self.consume() + + return decode(rv) + + def expr_or_value_state(self): + if self.peek(3) == "if ": + self.state = self.expr_state + else: + self.state = self.value_state + + def expr_state(self): + self.skip_whitespace() + c = self.char() + if c == eol: + raise ParseError(self.filename, self.line_number, "EOL in expression") + elif c in "'\"": + self.consume() + yield (token_types.string, self.consume_string(c)) + elif c == "#": + raise ParseError(self.filename, self.line_number, "Comment before end of expression") + elif c == ":": + yield (token_types.separator, c) + self.consume() + self.state = self.value_state + elif c in parens: + self.consume() + yield (token_types.paren, c) + elif c in ("!", "="): + self.state = self.operator_state + elif c in digits: + self.state = self.digit_state + else: + self.state = self.ident_state + + def operator_state(self): + # Only symbolic operators + index_0 = self.index + while True: + c = self.char() + if c == eol: + break + elif c in operator_chars: + self.consume() + else: + self.state = self.expr_state + break + yield (token_types.ident, self.line[index_0:self.index]) + + def digit_state(self): + index_0 = self.index + seen_dot = False + while True: + c = self.char() + if c == eol: + break + elif c in digits: + self.consume() + elif c == ".": + if seen_dot: + raise ParseError(self.filename, self.line_number, "Invalid number") + self.consume() + seen_dot = True + elif c in parens: + break + elif c in operator_chars: + break + elif c == " ": + break + elif c == ":": + break + else: + raise ParseError(self.filename, self.line_number, "Invalid character in number") + + self.state = self.expr_state + yield (token_types.number, self.line[index_0:self.index]) + + def ident_state(self): + index_0 = self.index + while True: + c = self.char() + if c == eol: + break + elif c == ".": + break + elif c in parens: + break + elif c in operator_chars: + break + elif c == " ": + break + elif c == ":": + break + else: + self.consume() + self.state = self.expr_state + yield (token_types.ident, self.line[index_0:self.index]) + + def consume_escape(self): + assert self.char() == "\\" + self.consume() + c = self.char() + self.consume() + if c == "x": + return self.decode_escape(2) + elif c == "u": + return self.decode_escape(4) + elif c == "U": + return self.decode_escape(6) + elif c in ["a", "b", "f", "n", "r", "t", "v"]: + return eval("'\%s'" % c) + elif c is eol: + raise ParseError(self.filename, self.line_number, "EOL in escape") + else: + return c + + def decode_escape(self, length): + value = 0 + for i in xrange(length): + c = self.char() + value *= 16 + value += self.escape_value(c) + self.consume() + + return unichr(value).encode("utf8") + + def escape_value(self, c): + if '0' <= c <= '9': + return ord(c) - ord('0') + elif 'a' <= c <= 'f': + return ord(c) - ord('a') + 10 + elif 'A' <= c <= 'F': + return ord(c) - ord('A') + 10 + else: + raise ParseError(self.filename, self.line_number, "Invalid character escape") + + +class Parser(object): + def __init__(self): + self.reset() + + def reset(self): + self.token = None + self.unary_operators = "!" + self.binary_operators = frozenset(["&&", "||", "=="]) + self.tokenizer = Tokenizer() + self.token_generator = None + self.tree = Treebuilder(DataNode(None)) + self.expr_builder = None + self.expr_builders = [] + + def parse(self, input): + self.reset() + self.token_generator = self.tokenizer.tokenize(input) + self.consume() + self.manifest() + return self.tree.node + + def consume(self): + self.token = self.token_generator.next() + + def expect(self, type, value=None): + if self.token[0] != type: + raise ParseError + if value is not None: + if self.token[1] != value: + raise ParseError + + self.consume() + + def manifest(self): + self.data_block() + self.expect(token_types.eof) + + def data_block(self): + while self.token[0] == token_types.string: + self.tree.append(KeyValueNode(self.token[1])) + self.consume() + self.expect(token_types.separator) + self.value_block() + self.tree.pop() + + while self.token == (token_types.paren, "["): + self.consume() + if self.token[0] != token_types.string: + raise ParseError + self.tree.append(DataNode(self.token[1])) + self.consume() + self.expect(token_types.paren, "]") + if self.token[0] == token_types.group_start: + self.consume() + self.data_block() + self.eof_or_end_group() + self.tree.pop() + + def eof_or_end_group(self): + if self.token[0] != token_types.eof: + self.expect(token_types.group_end) + + def value_block(self): + if self.token[0] == token_types.list_start: + self.consume() + self.list_value() + elif self.token[0] == token_types.string: + self.value() + elif self.token[0] == token_types.group_start: + self.consume() + self.expression_values() + if self.token[0] == token_types.string: + self.value() + self.eof_or_end_group() + elif self.token[0] == token_types.atom: + self.atom() + else: + raise ParseError + + def list_value(self): + self.tree.append(ListNode()) + while self.token[0] in (token_types.atom, token_types.string): + if self.token[0] == token_types.atom: + self.atom() + else: + self.value() + self.expect(token_types.list_end) + self.tree.pop() + + def expression_values(self): + while self.token == (token_types.ident, "if"): + self.consume() + self.tree.append(ConditionalNode()) + self.expr_start() + self.expect(token_types.separator) + if self.token[0] == token_types.string: + self.value() + else: + raise ParseError + self.tree.pop() + + def value(self): + self.tree.append(ValueNode(self.token[1])) + self.consume() + self.tree.pop() + + def atom(self): + if self.token[1] not in atoms: + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised symbol @%s" % self.token[1]) + self.tree.append(AtomNode(atoms[self.token[1]])) + self.consume() + self.tree.pop() + + def expr_start(self): + self.expr_builder = ExpressionBuilder(self.tokenizer) + self.expr_builders.append(self.expr_builder) + self.expr() + expression = self.expr_builder.finish() + self.expr_builders.pop() + self.expr_builder = self.expr_builders[-1] if self.expr_builders else None + if self.expr_builder: + self.expr_builder.operands[-1].children[-1].append(expression) + else: + self.tree.append(expression) + self.tree.pop() + + def expr(self): + self.expr_operand() + while (self.token[0] == token_types.ident and self.token[1] in binary_operators): + self.expr_bin_op() + self.expr_operand() + + def expr_operand(self): + if self.token == (token_types.paren, "("): + self.consume() + self.expr_builder.left_paren() + self.expr() + self.expect(token_types.paren, ")") + self.expr_builder.right_paren() + elif self.token[0] == token_types.ident and self.token[1] in unary_operators: + self.expr_unary_op() + self.expr_operand() + elif self.token[0] in [token_types.string, token_types.ident]: + self.expr_value() + elif self.token[0] == token_types.number: + self.expr_number() + else: + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised operand") + + def expr_unary_op(self): + if self.token[1] in unary_operators: + self.expr_builder.push_operator(UnaryOperatorNode(self.token[1])) + self.consume() + else: + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected unary operator") + + def expr_bin_op(self): + if self.token[1] in binary_operators: + self.expr_builder.push_operator(BinaryOperatorNode(self.token[1])) + self.consume() + else: + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected binary operator") + + def expr_value(self): + node_type = {token_types.string: StringNode, + token_types.ident: VariableNode}[self.token[0]] + self.expr_builder.push_operand(node_type(self.token[1])) + self.consume() + if self.token == (token_types.paren, "["): + self.consume() + self.expr_builder.operands[-1].append(IndexNode()) + self.expr_start() + self.expect(token_types.paren, "]") + + def expr_number(self): + self.expr_builder.push_operand(NumberNode(self.token[1])) + self.consume() + + +class Treebuilder(object): + def __init__(self, root): + self.root = root + self.node = root + + def append(self, node): + self.node.append(node) + self.node = node + return node + + def pop(self): + node = self.node + self.node = self.node.parent + return node + + +class ExpressionBuilder(object): + def __init__(self, tokenizer): + self.operands = [] + self.operators = [None] + self.tokenizer = tokenizer + + def finish(self): + while self.operators[-1] is not None: + self.pop_operator() + rv = self.pop_operand() + assert self.is_empty() + return rv + + def left_paren(self): + self.operators.append(None) + + def right_paren(self): + while self.operators[-1] is not None: + self.pop_operator() + if not self.operators: + raise ParseError(self.tokenizer.filename, self.tokenizer.line, + "Unbalanced parens") + + assert self.operators.pop() is None + + def push_operator(self, operator): + assert operator is not None + while self.precedence(self.operators[-1]) > self.precedence(operator): + self.pop_operator() + + self.operators.append(operator) + + def pop_operator(self): + operator = self.operators.pop() + if isinstance(operator, BinaryOperatorNode): + operand_1 = self.operands.pop() + operand_0 = self.operands.pop() + self.operands.append(BinaryExpressionNode(operator, operand_0, operand_1)) + else: + operand_0 = self.operands.pop() + self.operands.append(UnaryExpressionNode(operator, operand_0)) + + def push_operand(self, node): + self.operands.append(node) + + def pop_operand(self): + return self.operands.pop() + + def is_empty(self): + return len(self.operands) == 0 and all(item is None for item in self.operators) + + def precedence(self, operator): + if operator is None: + return 0 + return precedence(operator) + + +def parse(stream): + p = Parser() + return p.parse(stream) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/serializer.py b/testing/web-platform/harness/wptrunner/wptmanifest/serializer.py new file mode 100644 index 000000000..efa839d8d --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/serializer.py @@ -0,0 +1,140 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from node import NodeVisitor, ValueNode, ListNode, BinaryExpressionNode +from parser import atoms, precedence + +atom_names = {v:"@%s" % k for (k,v) in atoms.iteritems()} + +named_escapes = set(["\a", "\b", "\f", "\n", "\r", "\t", "\v"]) + +def escape(string, extras=""): + rv = "" + for c in string: + if c in named_escapes: + rv += c.encode("unicode_escape") + elif c == "\\": + rv += "\\\\" + elif c < '\x20': + rv += "\\x%02x" % ord(c) + elif c in extras: + rv += "\\" + c + else: + rv += c + return rv.encode("utf8") + + +class ManifestSerializer(NodeVisitor): + def __init__(self, skip_empty_data=False): + self.skip_empty_data = skip_empty_data + + def serialize(self, root): + self.indent = 2 + rv = "\n".join(self.visit(root)) + if rv[-1] != "\n": + rv = rv + "\n" + return rv + + def visit_DataNode(self, node): + rv = [] + if not self.skip_empty_data or node.children: + if node.data: + rv.append("[%s]" % escape(node.data, extras="]")) + indent = self.indent * " " + else: + indent = "" + + for child in node.children: + rv.extend("%s%s" % (indent if item else "", item) for item in self.visit(child)) + + if node.parent: + rv.append("") + + return rv + + def visit_KeyValueNode(self, node): + rv = [escape(node.data, ":") + ":"] + indent = " " * self.indent + + if len(node.children) == 1 and isinstance(node.children[0], (ValueNode, ListNode)): + rv[0] += " %s" % self.visit(node.children[0])[0] + else: + for child in node.children: + rv.append(indent + self.visit(child)[0]) + + return rv + + def visit_ListNode(self, node): + rv = ["["] + rv.extend(", ".join(self.visit(child)[0] for child in node.children)) + rv.append("]") + return ["".join(rv)] + + def visit_ValueNode(self, node): + if "#" in node.data or (isinstance(node.parent, ListNode) and + ("," in node.data or "]" in node.data)): + if "\"" in node.data: + quote = "'" + else: + quote = "\"" + else: + quote = "" + return [quote + escape(node.data, extras=quote) + quote] + + def visit_AtomNode(self, node): + return [atom_names[node.data]] + + def visit_ConditionalNode(self, node): + return ["if %s: %s" % tuple(self.visit(item)[0] for item in node.children)] + + def visit_StringNode(self, node): + rv = ["\"%s\"" % escape(node.data, extras="\"")] + for child in node.children: + rv[0] += self.visit(child)[0] + return rv + + def visit_NumberNode(self, node): + return [str(node.data)] + + def visit_VariableNode(self, node): + rv = escape(node.data) + for child in node.children: + rv += self.visit(child) + return [rv] + + def visit_IndexNode(self, node): + assert len(node.children) == 1 + return ["[%s]" % self.visit(node.children[0])[0]] + + def visit_UnaryExpressionNode(self, node): + children = [] + for child in node.children: + child_str = self.visit(child)[0] + if isinstance(child, BinaryExpressionNode): + child_str = "(%s)" % child_str + children.append(child_str) + return [" ".join(children)] + + def visit_BinaryExpressionNode(self, node): + assert len(node.children) == 3 + children = [] + for child_index in [1, 0, 2]: + child = node.children[child_index] + child_str = self.visit(child)[0] + if (isinstance(child, BinaryExpressionNode) and + precedence(node.children[0]) < precedence(child.children[0])): + child_str = "(%s)" % child_str + children.append(child_str) + return [" ".join(children)] + + def visit_UnaryOperatorNode(self, node): + return [str(node.data)] + + def visit_BinaryOperatorNode(self, node): + return [str(node.data)] + + +def serialize(tree, *args, **kwargs): + s = ManifestSerializer(*args, **kwargs) + return s.serialize(tree) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/__init__.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/__init__.py new file mode 100644 index 000000000..c580d191c --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py new file mode 100644 index 000000000..af18f4acc --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from cStringIO import StringIO + +from ..backends import conditional +from ..node import BinaryExpressionNode, BinaryOperatorNode, VariableNode, NumberNode + + +class TestConditional(unittest.TestCase): + def parse(self, input_str): + return self.parser.parse(StringIO(input_str)) + + def compile(self, input_text): + return conditional.compile(input_text) + + def test_get_0(self): + data = """ +key: value + +[Heading 1] + other_key: + if a == 1: value_1 + if a == 2: value_2 + value_3 +""" + + manifest = self.compile(data) + + self.assertEquals(manifest.get("key"), "value") + children = list(item for item in manifest.iterchildren()) + self.assertEquals(len(children), 1) + section = children[0] + self.assertEquals(section.name, "Heading 1") + + self.assertEquals(section.get("other_key", {"a": 1}), "value_1") + self.assertEquals(section.get("other_key", {"a": 2}), "value_2") + self.assertEquals(section.get("other_key", {"a": 7}), "value_3") + self.assertEquals(section.get("key"), "value") + + def test_get_1(self): + data = """ +key: value + +[Heading 1] + other_key: + if a == "1": value_1 + if a == 2: value_2 + value_3 +""" + + manifest = self.compile(data) + + children = list(item for item in manifest.iterchildren()) + section = children[0] + + self.assertEquals(section.get("other_key", {"a": "1"}), "value_1") + self.assertEquals(section.get("other_key", {"a": 1}), "value_3") + + def test_get_2(self): + data = """ +key: + if a[1] == "b": value_1 + if a[1] == 2: value_2 + value_3 +""" + + manifest = self.compile(data) + + self.assertEquals(manifest.get("key", {"a": "ab"}), "value_1") + self.assertEquals(manifest.get("key", {"a": [1, 2]}), "value_2") + + def test_get_3(self): + data = """ +key: + if a[1] == "ab"[1]: value_1 + if a[1] == 2: value_2 + value_3 +""" + + manifest = self.compile(data) + + self.assertEquals(manifest.get("key", {"a": "ab"}), "value_1") + self.assertEquals(manifest.get("key", {"a": [1, 2]}), "value_2") + + def test_set_0(self): + data = """ +key: + if a == "a": value_1 + if a == "b": value_2 + value_3 +""" + manifest = self.compile(data) + + manifest.set("new_key", "value_new") + + self.assertEquals(manifest.get("new_key"), "value_new") + + def test_set_1(self): + data = """ +key: + if a == "a": value_1 + if a == "b": value_2 + value_3 +""" + + manifest = self.compile(data) + + manifest.set("key", "value_new") + + self.assertEquals(manifest.get("key"), "value_new") + self.assertEquals(manifest.get("key", {"a": "a"}), "value_1") + + def test_set_2(self): + data = """ +key: + if a == "a": value_1 + if a == "b": value_2 + value_3 +""" + + manifest = self.compile(data) + + expr = BinaryExpressionNode(BinaryOperatorNode("=="), + VariableNode("a"), + NumberNode("1")) + + manifest.set("key", "value_new", expr) + + self.assertEquals(manifest.get("key", {"a": 1}), "value_new") + self.assertEquals(manifest.get("key", {"a": "a"}), "value_1") + + def test_api_0(self): + data = """ +key: + if a == 1.5: value_1 + value_2 +key_1: other_value +""" + manifest = self.compile(data) + + self.assertFalse(manifest.is_empty) + self.assertEquals(manifest.root, manifest) + self.assertTrue(manifest.has_key("key_1")) + self.assertFalse(manifest.has_key("key_2")) + + self.assertEquals(set(manifest.iterkeys()), set(["key", "key_1"])) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_parser.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_parser.py new file mode 100644 index 000000000..6e8e6e6be --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_parser.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from cStringIO import StringIO + +from .. import parser + +# There aren't many tests here because it turns out to be way more convenient to +# use test_serializer for the majority of cases + + +class TestExpression(unittest.TestCase): + def setUp(self): + self.parser = parser.Parser() + + def parse(self, input_str): + return self.parser.parse(StringIO(input_str)) + + def compare(self, input_text, expected): + actual = self.parse(input_text) + self.match(expected, actual) + + def match(self, expected_node, actual_node): + self.assertEquals(expected_node[0], actual_node.__class__.__name__) + self.assertEquals(expected_node[1], actual_node.data) + self.assertEquals(len(expected_node[2]), len(actual_node.children)) + for expected_child, actual_child in zip(expected_node[2], actual_node.children): + self.match(expected_child, actual_child) + + def test_expr_0(self): + self.compare( + """ +key: + if x == 1 : value""", + ["DataNode", None, + [["KeyValueNode", "key", + [["ConditionalNode", None, + [["BinaryExpressionNode", None, + [["BinaryOperatorNode", "==", []], + ["VariableNode", "x", []], + ["NumberNode", "1", []] + ]], + ["ValueNode", "value", []], + ]]]]]] + ) + + def test_expr_1(self): + self.compare( + """ +key: + if not x and y : value""", + ["DataNode", None, + [["KeyValueNode", "key", + [["ConditionalNode", None, + [["BinaryExpressionNode", None, + [["BinaryOperatorNode", "and", []], + ["UnaryExpressionNode", None, + [["UnaryOperatorNode", "not", []], + ["VariableNode", "x", []] + ]], + ["VariableNode", "y", []] + ]], + ["ValueNode", "value", []], + ]]]]]] + ) + + def test_atom_0(self): + with self.assertRaises(parser.ParseError): + self.parse("key: @Unknown") + + def test_atom_1(self): + with self.assertRaises(parser.ParseError): + self.parse("key: @true") + +if __name__ == "__main__": + unittest.main() diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py new file mode 100644 index 000000000..ec4d6e2d7 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py @@ -0,0 +1,227 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from cStringIO import StringIO + +from .. import parser, serializer + + +class TokenizerTest(unittest.TestCase): + def setUp(self): + self.serializer = serializer.ManifestSerializer() + self.parser = parser.Parser() + + def serialize(self, input_str): + return self.serializer.serialize(self.parser.parse(input_str)) + + def compare(self, input_str, expected=None): + if expected is None: + expected = input_str + expected = expected.encode("utf8") + actual = self.serialize(input_str) + self.assertEquals(actual, expected) + + def test_0(self): + self.compare("""key: value +[Heading 1] + other_key: other_value +""") + + def test_1(self): + self.compare("""key: value +[Heading 1] + other_key: + if a or b: other_value +""") + + def test_2(self): + self.compare("""key: value +[Heading 1] + other_key: + if a or b: other_value + fallback_value +""") + + def test_3(self): + self.compare("""key: value +[Heading 1] + other_key: + if a == 1: other_value + fallback_value +""") + + def test_4(self): + self.compare("""key: value +[Heading 1] + other_key: + if a == "1": other_value + fallback_value +""") + + def test_5(self): + self.compare("""key: value +[Heading 1] + other_key: + if a == "abc"[1]: other_value + fallback_value +""") + + def test_6(self): + self.compare("""key: value +[Heading 1] + other_key: + if a == "abc"[c]: other_value + fallback_value +""") + + def test_7(self): + self.compare("""key: value +[Heading 1] + other_key: + if (a or b) and c: other_value + fallback_value +""", +"""key: value +[Heading 1] + other_key: + if a or b and c: other_value + fallback_value +""") + + def test_8(self): + self.compare("""key: value +[Heading 1] + other_key: + if a or (b and c): other_value + fallback_value +""") + + def test_9(self): + self.compare("""key: value +[Heading 1] + other_key: + if not (a and b): other_value + fallback_value +""") + + def test_10(self): + self.compare("""key: value +[Heading 1] + some_key: some_value + +[Heading 2] + other_key: other_value +""") + + def test_11(self): + self.compare("""key: + if not a and b and c and d: true +""") + + def test_12(self): + self.compare("""[Heading 1] + key: [a:1, b:2] +""") + + def test_13(self): + self.compare("""key: [a:1, "b:#"] +""") + + def test_14(self): + self.compare("""key: [","] +""") + + def test_15(self): + self.compare("""key: , +""") + + def test_16(self): + self.compare("""key: ["]", b] +""") + + def test_17(self): + self.compare("""key: ] +""") + + def test_18(self): + self.compare("""key: \] + """, """key: ] +""") + + def test_escape_0(self): + self.compare(r"""k\t\:y: \a\b\f\n\r\t\v""", + r"""k\t\:y: \x07\x08\x0c\n\r\t\x0b +""") + + def test_escape_1(self): + self.compare(r"""k\x00: \x12A\x45""", + r"""k\x00: \x12AE +""") + + def test_escape_2(self): + self.compare(r"""k\u0045y: \u1234A\uABc6""", + u"""kEy: \u1234A\uabc6 +""") + + def test_escape_3(self): + self.compare(r"""k\u0045y: \u1234A\uABc6""", + u"""kEy: \u1234A\uabc6 +""") + + def test_escape_4(self): + self.compare(r"""key: '\u1234A\uABc6'""", + u"""key: \u1234A\uabc6 +""") + + def test_escape_5(self): + self.compare(r"""key: [\u1234A\uABc6]""", + u"""key: [\u1234A\uabc6] +""") + + def test_escape_6(self): + self.compare(r"""key: [\u1234A\uABc6\,]""", + u"""key: ["\u1234A\uabc6,"] +""") + + def test_escape_7(self): + self.compare(r"""key: [\,\]\#]""", + r"""key: [",]#"] +""") + + def test_escape_8(self): + self.compare(r"""key: \#""", + r"""key: "#" +""") + + def test_escape_9(self): + self.compare(r"""key: \U10FFFFabc""", + u"""key: \U0010FFFFabc +""") + + def test_escape_10(self): + self.compare(r"""key: \u10FFab""", + u"""key: \u10FFab +""") + + def test_escape_11(self): + self.compare(r"""key: \\ab +""") + + def test_atom_1(self): + self.compare(r"""key: @True +""") + + def test_atom_2(self): + self.compare(r"""key: @False +""") + + def test_atom_3(self): + self.compare(r"""key: @Reset +""") + + def test_atom_4(self): + self.compare(r"""key: [a, @Reset, b] +""") diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py new file mode 100644 index 000000000..5bd270d9f --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from cStringIO import StringIO + +from ..backends import static + +# There aren't many tests here because it turns out to be way more convenient to +# use test_serializer for the majority of cases + + +class TestStatic(unittest.TestCase): + def parse(self, input_str): + return self.parser.parse(StringIO(input_str)) + + def compile(self, input_text, input_data): + return static.compile(input_text, input_data) + + def test_get_0(self): + data = """ +key: value + +[Heading 1] + other_key: + if a == 1: value_1 + if a == 2: value_2 + value_3 +""" + + manifest = self.compile(data, {"a": 2}) + + self.assertEquals(manifest.get("key"), "value") + children = list(item for item in manifest.iterchildren()) + self.assertEquals(len(children), 1) + section = children[0] + self.assertEquals(section.name, "Heading 1") + + self.assertEquals(section.get("other_key"), "value_2") + self.assertEquals(section.get("key"), "value") + + def test_get_1(self): + data = """ +key: value + +[Heading 1] + other_key: + if a == 1: value_1 + if a == 2: value_2 + value_3 +""" + manifest = self.compile(data, {"a": 3}) + + children = list(item for item in manifest.iterchildren()) + section = children[0] + self.assertEquals(section.get("other_key"), "value_3") + + def test_get_3(self): + data = """key: + if a == "1": value_1 + if a[0] == "ab"[0]: value_2 +""" + manifest = self.compile(data, {"a": "1"}) + self.assertEquals(manifest.get("key"), "value_1") + + manifest = self.compile(data, {"a": "ac"}) + self.assertEquals(manifest.get("key"), "value_2") + + def test_get_4(self): + data = """key: + if not a: value_1 + value_2 +""" + manifest = self.compile(data, {"a": True}) + self.assertEquals(manifest.get("key"), "value_2") + + manifest = self.compile(data, {"a": False}) + self.assertEquals(manifest.get("key"), "value_1") + + def test_api(self): + data = """key: + if a == 1.5: value_1 + value_2 +key_1: other_value +""" + manifest = self.compile(data, {"a": 1.5}) + + self.assertFalse(manifest.is_empty) + self.assertEquals(manifest.root, manifest) + self.assertTrue(manifest.has_key("key_1")) + self.assertFalse(manifest.has_key("key_2")) + + self.assertEquals(set(manifest.iterkeys()), set(["key", "key_1"])) + self.assertEquals(set(manifest.itervalues()), set(["value_1", "other_value"])) + + def test_is_empty_1(self): + data = """ +[Section] + [Subsection] +""" + manifest = self.compile(data, {}) + + self.assertTrue(manifest.is_empty) diff --git a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_tokenizer.py b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_tokenizer.py new file mode 100644 index 000000000..939567713 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_tokenizer.py @@ -0,0 +1,361 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import os +import unittest + +sys.path.insert(0, os.path.abspath("..")) +from cStringIO import StringIO + +from .. import parser +from ..parser import token_types + + +class TokenizerTest(unittest.TestCase): + def setUp(self): + self.tokenizer = parser.Tokenizer() + + def tokenize(self, input_str): + rv = [] + for item in self.tokenizer.tokenize(StringIO(input_str)): + rv.append(item) + if item[0] == token_types.eof: + break + return rv + + def compare(self, input_text, expected): + expected = expected + [(token_types.eof, None)] + actual = self.tokenize(input_text) + self.assertEquals(actual, expected) + + def test_heading_0(self): + self.compare("""[Heading text]""", + [(token_types.paren, "["), + (token_types.string, "Heading text"), + (token_types.paren, "]")]) + + def test_heading_1(self): + self.compare("""[Heading [text\]]""", + [(token_types.paren, "["), + (token_types.string, "Heading [text]"), + (token_types.paren, "]")]) + + def test_heading_2(self): + self.compare("""[Heading #text]""", + [(token_types.paren, "["), + (token_types.string, "Heading #text"), + (token_types.paren, "]")]) + + def test_heading_3(self): + self.compare("""[Heading [\]text]""", + [(token_types.paren, "["), + (token_types.string, "Heading []text"), + (token_types.paren, "]")]) + + def test_heading_4(self): + with self.assertRaises(parser.ParseError): + self.tokenize("[Heading") + + def test_heading_5(self): + self.compare("""[Heading [\]text] #comment""", + [(token_types.paren, "["), + (token_types.string, "Heading []text"), + (token_types.paren, "]")]) + + def test_heading_6(self): + self.compare(r"""[Heading \ttext]""", + [(token_types.paren, "["), + (token_types.string, "Heading \ttext"), + (token_types.paren, "]")]) + + def test_key_0(self): + self.compare("""key:value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_key_1(self): + self.compare("""key : value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_key_2(self): + self.compare("""key : val ue""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "val ue")]) + + def test_key_3(self): + self.compare("""key: value#comment""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_key_4(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""ke y: value""") + + def test_key_5(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key""") + + def test_key_6(self): + self.compare("""key: "value\"""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_key_7(self): + self.compare("""key: 'value'""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_key_8(self): + self.compare("""key: "#value\"""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "#value")]) + + def test_key_9(self): + self.compare("""key: '#value\'""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, "#value")]) + + def test_key_10(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: "value""") + + def test_key_11(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: 'value""") + + def test_key_12(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: 'value""") + + def test_key_13(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: 'value' abc""") + + def test_key_14(self): + self.compare(r"""key: \\nb""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.string, r"\nb")]) + + def test_list_0(self): + self.compare( +""" +key: []""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.list_start, "["), + (token_types.list_end, "]")]) + + def test_list_1(self): + self.compare( +""" +key: [a, "b"]""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.list_start, "["), + (token_types.string, "a"), + (token_types.string, "b"), + (token_types.list_end, "]")]) + + def test_list_2(self): + self.compare( +""" +key: [a, + b]""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.list_start, "["), + (token_types.string, "a"), + (token_types.string, "b"), + (token_types.list_end, "]")]) + + def test_list_3(self): + self.compare( +""" +key: [a, #b] + c]""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.list_start, "["), + (token_types.string, "a"), + (token_types.string, "c"), + (token_types.list_end, "]")]) + + def test_list_4(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: [a #b] + c]""") + + def test_list_5(self): + with self.assertRaises(parser.ParseError): + self.tokenize("""key: [a \\ + c]""") + + def test_list_6(self): + self.compare( +"""key: [a , b]""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.list_start, "["), + (token_types.string, "a"), + (token_types.string, "b"), + (token_types.list_end, "]")]) + + def test_expr_0(self): + self.compare( +""" +key: + if cond == 1: value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.number, "1"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_1(self): + self.compare( +""" +key: + if cond == 1: value1 + value2""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.number, "1"), + (token_types.separator, ":"), + (token_types.string, "value1"), + (token_types.string, "value2")]) + + def test_expr_2(self): + self.compare( +""" +key: + if cond=="1": value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.string, "1"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_3(self): + self.compare( +""" +key: + if cond==1.1: value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.number, "1.1"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_4(self): + self.compare( + """ +key: + if cond==1.1 and cond2 == "a": value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.number, "1.1"), + (token_types.ident, "and"), + (token_types.ident, "cond2"), + (token_types.ident, "=="), + (token_types.string, "a"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_5(self): + self.compare( +""" +key: + if (cond==1.1 ): value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.paren, "("), + (token_types.ident, "cond"), + (token_types.ident, "=="), + (token_types.number, "1.1"), + (token_types.paren, ")"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_6(self): + self.compare( +""" +key: + if "\\ttest": value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.string, "\ttest"), + (token_types.separator, ":"), + (token_types.string, "value")]) + + def test_expr_7(self): + with self.assertRaises(parser.ParseError): + self.tokenize( +""" +key: + if 1A: value""") + + def test_expr_8(self): + with self.assertRaises(parser.ParseError): + self.tokenize( +""" +key: + if 1a: value""") + + def test_expr_9(self): + with self.assertRaises(parser.ParseError): + self.tokenize( +""" +key: + if 1.1.1: value""") + + def test_expr_10(self): + self.compare( +""" +key: + if 1.: value""", + [(token_types.string, "key"), + (token_types.separator, ":"), + (token_types.group_start, None), + (token_types.ident, "if"), + (token_types.number, "1."), + (token_types.separator, ":"), + (token_types.string, "value")]) + +if __name__ == "__main__": + unittest.main() |