#!/usr/bin/env python2.4 """usage: %progname candidate_path baseline_path """ from __future__ import print_function import json import optparse from operator import itemgetter def avg(seq): return sum(seq) / len(seq) def compare(current, baseline): percent_speedups = [] for key, current_result in current.iteritems(): try: baseline_result = baseline[key] except KeyError: print(key, 'missing from baseline') continue val_getter = itemgetter('average_ms', 'stddev_ms') base_avg, base_stddev = val_getter(baseline_result) current_avg, current_stddev = val_getter(current_result) t_best = current_avg - current_stddev t_worst = current_avg + current_stddev base_t_best = base_avg - base_stddev base_t_worst = base_avg + base_stddev if t_worst < base_t_best: # Worst takes less time (better) than baseline's best. speedup = -((t_worst - base_t_best) / base_t_best) * 100 result = 'faster: {:6.2f}ms < baseline {:6.2f}ms ({:+6.2f}%)'.format( t_worst, base_t_best, speedup) percent_speedups.append(speedup) elif t_best > base_t_worst: # Best takes more time (worse) than baseline's worst. slowdown = -((t_best - base_t_worst) / base_t_worst) * 100 result = 'SLOWER: {:6.2f}ms > baseline {:6.2f}ms ({:+6.2f}%) '.format( t_best, base_t_worst, slowdown) percent_speedups.append(slowdown) else: result = 'Meh.' print('{:30s}: {}'.format(key, result)) if percent_speedups: print('Average speedup: {:.2f}%'.format(avg(percent_speedups))) def compare_immediate(current_map, baseline_path): baseline_file = open(baseline_path) baseline_map = json.load(baseline_file) baseline_file.close() compare(current_map, baseline_map) def main(candidate_path, baseline_path): candidate_file, baseline_file = open(candidate_path), open(baseline_path) candidate = json.load(candidate_file) baseline = json.load(baseline_file) compare(candidate, baseline) candidate_file.close() baseline_file.close() if __name__ == '__main__': parser = optparse.OptionParser(usage=__doc__.strip()) options, args = parser.parse_args() try: candidate_path = args.pop(0) except IndexError: parser.error('A JSON filepath to compare against baseline is required') try: baseline_path = args.pop(0) except IndexError: parser.error('A JSON filepath for baseline is required') main(candidate_path, baseline_path)