1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
|
from __future__ import print_function
import argparse
import json
import sys
from cgi import escape
from collections import defaultdict
import types
def html_escape(item, escape_quote=False):
if isinstance(item, types.StringTypes):
rv = escape(item)
if escape_quote:
rv = rv.replace('"', """)
return rv
else:
return item
class Raw(object):
"""Simple wrapper around a string to stop it being escaped by html_escape"""
def __init__(self, value):
self.value = value
def __unicode__(self):
return unicode(self.value)
class Node(object):
"""Node structure used when building HTML"""
def __init__(self, name, attrs, children):
#Need list of void elements
self.name = name
self.attrs = attrs
self.children = children
def __unicode__(self):
if self.attrs:
#Need to escape
attrs_unicode = " " + " ".join("%s=\"%s\"" % (html_escape(key),
html_escape(value,
escape_quote=True))
for key, value in self.attrs.iteritems())
else:
attrs_unicode = ""
return "<%s%s>%s</%s>\n" % (self.name,
attrs_unicode,
"".join(unicode(html_escape(item))
for item in self.children),
self.name)
def __str__(self):
return unicode(self).encode("utf8")
class RootNode(object):
"""Special Node representing the document root"""
def __init__(self, *children):
self.children = ["<!DOCTYPE html>"] + list(children)
def __unicode__(self):
return "".join(unicode(item) for item in self.children)
def __str__(self):
return unicode(self).encode("utf8")
def flatten(iterable):
"""Flatten a list of lists by one level so that
[1,["abc"], "def",[2, [3]]]
becomes
[1, "abc", "def", 2, [3]]"""
rv = []
for item in iterable:
if hasattr(item, "__iter__") and not isinstance(item, types.StringTypes):
rv.extend(item)
else:
rv.append(item)
return rv
class HTML(object):
"""Simple HTML templating system. An instance of this class can create
element nodes by calling methods with the same name as the element,
passing in children as positional arguments or as a list, and attributes
as keyword arguments, with _ replacing - and trailing _ for python keywords
e.g.
h = HTML()
print(h.html(
html.head(),
html.body([html.h1("Hello World!")], class_="body-class")
))
Would give
<!DOCTYPE html><html><head></head><body class="body-class"><h1>Hello World!</h1></body></html>"""
def __getattr__(self, name):
def make_html(self, *content, **attrs):
for attr_name in attrs.keys():
if "_" in attr_name:
new_name = attr_name.replace("_", "-")
if new_name.endswith("-"):
new_name = new_name[:-1]
attrs[new_name] = attrs.pop(attr_name)
return Node(name, attrs, flatten(content))
method = types.MethodType(make_html, self, HTML)
setattr(self, name, method)
return method
def __call__(self, *children):
return RootNode(*flatten(children))
h = HTML()
class TestResult(object):
"""Simple holder for the results of a single test in a single UA"""
def __init__(self, test):
self.test = test
self.results = {}
def __cmp__(self, other):
return self.test == other.test
def __hash__(self):
return hash(self.test)
def load_data(args):
"""Load data treating args as a list of UA name, filename pairs"""
pairs = []
for i in xrange(0, len(args), 2):
pairs.append(args[i:i+2])
rv = {}
for UA, filename in pairs:
with open(filename) as f:
rv[UA] = json.load(f)
return rv
def test_id(id):
"""Convert a test id in JSON into an immutable object that
can be used as a dictionary key"""
if isinstance(id, list):
return tuple(id)
else:
return id
def all_tests(data):
tests = defaultdict(set)
for UA, results in data.iteritems():
for result in results["results"]:
id = test_id(result["test"])
tests[id] |= set(subtest["name"] for subtest in result["subtests"])
return tests
def group_results(data):
"""Produce a list of UAs and a dictionary mapping specific tests to their
status in all UAs e.g.
["UA1", "UA2"], {"test_id":{"harness":{"UA1": (status1, message1),
"UA2": (status2, message2)},
"subtests":{"subtest1": "UA1": (status1-1, message1-1),
"UA2": (status2-1, message2-1)}}}
Status and message are None if the test didn't run in a particular UA.
Message is None if the test didn't produce a message"""
tests = all_tests(data)
UAs = data.keys()
def result():
return {
"harness": dict((UA, (None, None)) for UA in UAs),
"subtests": None # init this later
}
results_by_test = defaultdict(result)
for UA, results in data.iteritems():
for test_data in results["results"]:
id = test_id(test_data["test"])
result = results_by_test[id]
if result["subtests"] is None:
result["subtests"] = dict(
(name, dict((UA, (None, None)) for UA in UAs)) for name in tests[id]
)
result["harness"][UA] = (test_data["status"], test_data["message"])
for subtest in test_data["subtests"]:
result["subtests"][subtest["name"]][UA] = (subtest["status"],
subtest["message"])
return UAs, results_by_test
def status_cell(status, message=None):
"""Produce a table cell showing the status of a test"""
status = status if status is not None else "NONE"
kwargs = {}
if message:
kwargs["title"] = message
status_text = status.title()
return h.td(status_text, class_="status " + status,
**kwargs)
def test_link(test_id, subtest=None):
"""Produce an <a> element linking to a test"""
if isinstance(test_id, types.StringTypes):
rv = [h.a(test_id, href=test_id)]
else:
rv = [h.a(test_id[0], href=test_id[0]),
" %s " % test_id[1],
h.a(test_id[2], href=test_id[2])]
if subtest is not None:
rv.append(" [%s]" % subtest)
return rv
def summary(UAs, results_by_test):
"""Render the implementation report summary"""
not_passing = []
for test, results in results_by_test.iteritems():
if not any(item[0] in ("PASS", "OK") for item in results["harness"].values()):
not_passing.append((test, None))
for subtest_name, subtest_results in results["subtests"].iteritems():
if not any(item[0] == "PASS" for item in subtest_results.values()):
not_passing.append((test, subtest_name))
if not_passing:
rv = [
h.p("The following tests failed to pass in all UAs:"),
h.ul([h.li(test_link(test, subtest))
for test, subtest in not_passing])
]
else:
rv = "All tests passed in at least one UA"
return rv
def result_rows(UAs, test, result):
"""Render the results for each test run"""
yield h.tr(
h.td(
test_link(test),
rowspan=(1 + len(result["subtests"]))
),
h.td(),
[status_cell(status, message)
for UA, (status, message) in sorted(result["harness"].items())],
class_="test"
)
for name, subtest_result in sorted(result["subtests"].iteritems()):
yield h.tr(
h.td(name),
[status_cell(status, message)
for UA, (status, message) in sorted(subtest_result.items())],
class_="subtest"
)
def result_bodies(UAs, results_by_test):
return [h.tbody(result_rows(UAs, test, result))
for test, result in sorted(results_by_test.iteritems())]
def generate_html(UAs, results_by_test):
"""Generate all the HTML output"""
return h(h.html(
h.head(
h.meta(charset="utf8"),
h.title("Implementation Report"),
h.link(href="report.css", rel="stylesheet")),
h.body(
h.h1("Implementation Report"),
h.h2("Summary"),
summary(UAs, results_by_test),
h.h2("Full Results"),
h.table(
h.thead(
h.tr(
h.th("Test"),
h.th("Subtest"),
[h.th(UA) for UA in sorted(UAs)])),
result_bodies(UAs, results_by_test)))))
def main(filenames):
data = load_data(filenames)
UAs, results_by_test = group_results(data)
return generate_html(UAs, results_by_test)
if __name__ == "__main__":
if not sys.argv[1:]:
print("""Please supply a list of UA name, filename pairs e.g.
python report.py Firefox firefox.json Chrome chrome.json IE internet_explorer.json""")
print(main(sys.argv[1:]))
|
