1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
import math
import mozinfo
class Bisect(object):
"Class for creating, bisecting and summarizing for --bisect-chunk option."
def __init__(self, harness):
super(Bisect, self).__init__()
self.summary = []
self.contents = {}
self.repeat = 10
self.failcount = 0
self.max_failures = 3
def setup(self, tests):
"""This method is used to initialize various variables that are required
for test bisection"""
status = 0
self.contents.clear()
# We need totalTests key in contents for sanity check
self.contents['totalTests'] = tests
self.contents['tests'] = tests
self.contents['loop'] = 0
return status
def reset(self, expectedError, result):
"""This method is used to initialize self.expectedError and self.result
for each loop in runtests."""
self.expectedError = expectedError
self.result = result
def get_tests_for_bisection(self, options, tests):
"""Make a list of tests for bisection from a given list of tests"""
bisectlist = []
for test in tests:
bisectlist.append(test)
if test.endswith(options.bisectChunk):
break
return bisectlist
def pre_test(self, options, tests, status):
"""This method is used to call other methods for setting up variables and
getting the list of tests for bisection."""
if options.bisectChunk == "default":
return tests
# The second condition in 'if' is required to verify that the failing
# test is the last one.
elif ('loop' not in self.contents or not self.contents['tests'][-1].endswith(
options.bisectChunk)):
tests = self.get_tests_for_bisection(options, tests)
status = self.setup(tests)
return self.next_chunk_binary(options, status)
def post_test(self, options, expectedError, result):
"""This method is used to call other methods to summarize results and check whether a
sanity check is done or not."""
self.reset(expectedError, result)
status = self.summarize_chunk(options)
# Check whether sanity check has to be done. Also it is necessary to check whether
# options.bisectChunk is present in self.expectedError as we do not want to run
# if it is "default".
if status == -1 and options.bisectChunk in self.expectedError:
# In case we have a debug build, we don't want to run a sanity
# check, will take too much time.
if mozinfo.info['debug']:
return status
testBleedThrough = self.contents['testsToRun'][0]
tests = self.contents['totalTests']
tests.remove(testBleedThrough)
# To make sure that the failing test is dependent on some other
# test.
if options.bisectChunk in testBleedThrough:
return status
status = self.setup(tests)
self.summary.append("Sanity Check:")
return status
def next_chunk_reverse(self, options, status):
"This method is used to bisect the tests in a reverse search fashion."
# Base Cases.
if self.contents['loop'] <= 1:
self.contents['testsToRun'] = self.contents['tests']
if self.contents['loop'] == 1:
self.contents['testsToRun'] = [self.contents['tests'][-1]]
self.contents['loop'] += 1
return self.contents['testsToRun']
if 'result' in self.contents:
if self.contents['result'] == "PASS":
chunkSize = self.contents['end'] - self.contents['start']
self.contents['end'] = self.contents['start'] - 1
self.contents['start'] = self.contents['end'] - chunkSize
# self.contents['result'] will be expected error only if it fails.
elif self.contents['result'] == "FAIL":
self.contents['tests'] = self.contents['testsToRun']
status = 1 # for initializing
# initialize
if status:
totalTests = len(self.contents['tests'])
chunkSize = int(math.ceil(totalTests / 10.0))
self.contents['start'] = totalTests - chunkSize - 1
self.contents['end'] = totalTests - 2
start = self.contents['start']
end = self.contents['end'] + 1
self.contents['testsToRun'] = self.contents['tests'][start:end]
self.contents['testsToRun'].append(self.contents['tests'][-1])
self.contents['loop'] += 1
return self.contents['testsToRun']
def next_chunk_binary(self, options, status):
"This method is used to bisect the tests in a binary search fashion."
# Base cases.
if self.contents['loop'] <= 1:
self.contents['testsToRun'] = self.contents['tests']
if self.contents['loop'] == 1:
self.contents['testsToRun'] = [self.contents['tests'][-1]]
self.contents['loop'] += 1
return self.contents['testsToRun']
# Initialize the contents dict.
if status:
totalTests = len(self.contents['tests'])
self.contents['start'] = 0
self.contents['end'] = totalTests - 2
mid = (self.contents['start'] + self.contents['end']) / 2
if 'result' in self.contents:
if self.contents['result'] == "PASS":
self.contents['end'] = mid
elif self.contents['result'] == "FAIL":
self.contents['start'] = mid + 1
mid = (self.contents['start'] + self.contents['end']) / 2
start = mid + 1
end = self.contents['end'] + 1
self.contents['testsToRun'] = self.contents['tests'][start:end]
if not self.contents['testsToRun']:
self.contents['testsToRun'].append(self.contents['tests'][mid])
self.contents['testsToRun'].append(self.contents['tests'][-1])
self.contents['loop'] += 1
return self.contents['testsToRun']
def summarize_chunk(self, options):
"This method is used summarize the results after the list of tests is run."
if options.bisectChunk == "default":
# if no expectedError that means all the tests have successfully
# passed.
if len(self.expectedError) == 0:
return -1
options.bisectChunk = self.expectedError.keys()[0]
self.summary.append(
"\tFound Error in test: %s" %
options.bisectChunk)
return 0
# If options.bisectChunk is not in self.result then we need to move to
# the next run.
if options.bisectChunk not in self.result:
return -1
self.summary.append("\tPass %d:" % self.contents['loop'])
if len(self.contents['testsToRun']) > 1:
self.summary.append(
"\t\t%d test files(start,end,failing). [%s, %s, %s]" % (len(
self.contents['testsToRun']),
self.contents['testsToRun'][0],
self.contents['testsToRun'][
-2],
self.contents['testsToRun'][
-1]))
else:
self.summary.append(
"\t\t1 test file [%s]" %
self.contents['testsToRun'][0])
return self.check_for_intermittent(options)
if self.result[options.bisectChunk] == "PASS":
self.summary.append("\t\tno failures found.")
if self.contents['loop'] == 1:
status = -1
else:
self.contents['result'] = "PASS"
status = 0
elif self.result[options.bisectChunk] == "FAIL":
if 'expectedError' not in self.contents:
self.summary.append("\t\t%s failed." %
self.contents['testsToRun'][-1])
self.contents['expectedError'] = self.expectedError[
options.bisectChunk]
status = 0
elif self.expectedError[options.bisectChunk] == self.contents['expectedError']:
self.summary.append(
"\t\t%s failed with expected error." % self.contents['testsToRun'][-1])
self.contents['result'] = "FAIL"
status = 0
# This code checks for test-bleedthrough. Should work for any
# algorithm.
numberOfTests = len(self.contents['testsToRun'])
if numberOfTests < 3:
# This means that only 2 tests are run. Since the last test
# is the failing test itself therefore the bleedthrough
# test is the first test
self.summary.append(
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
"root cause for many of the above failures" %
self.contents['testsToRun'][0])
status = -1
else:
self.summary.append(
"\t\t%s failed with different error." % self.contents['testsToRun'][-1])
status = -1
return status
def check_for_intermittent(self, options):
"This method is used to check whether a test is an intermittent."
if self.result[options.bisectChunk] == "PASS":
self.summary.append(
"\t\tThe test %s passed." %
self.contents['testsToRun'][0])
if self.repeat > 0:
# loop is set to 1 to again run the single test.
self.contents['loop'] = 1
self.repeat -= 1
return 0
else:
if self.failcount > 0:
# -1 is being returned as the test is intermittent, so no need to bisect
# further.
return -1
# If the test does not fail even once, then proceed to next chunk for bisection.
# loop is set to 2 to proceed on bisection.
self.contents['loop'] = 2
return 1
elif self.result[options.bisectChunk] == "FAIL":
self.summary.append(
"\t\tThe test %s failed." %
self.contents['testsToRun'][0])
self.failcount += 1
self.contents['loop'] = 1
self.repeat -= 1
# self.max_failures is the maximum number of times a test is allowed
# to fail to be called an intermittent. If a test fails more than
# limit set, it is a perma-fail.
if self.failcount < self.max_failures:
if self.repeat == 0:
# -1 is being returned as the test is intermittent, so no need to bisect
# further.
return -1
return 0
else:
self.summary.append(
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
"root cause for many of the above failures" %
self.contents['testsToRun'][0])
return -1
def print_summary(self):
"This method is used to print the recorded summary."
print "Bisection summary:"
for line in self.summary:
print line
|
