|
|
|
|
|
|
|
|
import subprocess, os, sys |
|
|
import xml.etree.ElementTree as ET |
|
|
from collections import defaultdict |
|
|
from statistics import median, stdev |
|
|
from datetime import datetime |
|
|
|
|
|
def get_commit_hash(): |
|
|
res = subprocess.run('git rev-parse HEAD'.split(), check=True, stdout=subprocess.PIPE, universal_newlines=True) |
|
|
return res.stdout.strip() |
|
|
|
|
|
if len(sys.argv) < 2: |
|
|
print('Usage: {} benchmark-binary'.format(sys.argv[0])) |
|
|
exit(1) |
|
|
|
|
|
|
|
|
num_runs = 10 |
|
|
data = defaultdict(list) |
|
|
|
|
|
|
|
|
def parse_file(file): |
|
|
|
|
|
def recursive_search(node): |
|
|
if node.tag == 'TestCase': |
|
|
results = node.find('OverallResult') |
|
|
time = results.get('durationInSeconds') |
|
|
data[node.get('name')].append(float(time)) |
|
|
elif node.tag in ('Group', 'Catch'): |
|
|
for child in node: |
|
|
recursive_search(child) |
|
|
|
|
|
tree = ET.parse(file) |
|
|
recursive_search(tree.getroot()) |
|
|
|
|
|
def run_benchmarks(binary): |
|
|
call = [binary] + '-d yes -r xml -o'.split() |
|
|
for i in range(num_runs): |
|
|
file = 'temp{}.xml'.format(i) |
|
|
print('Run number {}'.format(i)) |
|
|
subprocess.run(call + [file]) |
|
|
parse_file(file) |
|
|
|
|
|
os.remove(file) |
|
|
|
|
|
|
|
|
|
|
|
run_benchmarks(sys.argv[1]) |
|
|
|
|
|
result_file = '{:%Y-%m-%dT%H-%M-%S}-{}.result'.format(datetime.now(), get_commit_hash()) |
|
|
|
|
|
|
|
|
print('Writing results to {}'.format(result_file)) |
|
|
with open(result_file, 'w') as file: |
|
|
for k in sorted(data): |
|
|
file.write('{}: median: {} (s), stddev: {} (s)\n'.format(k, median(data[k]), stdev(data[k]))) |
|
|
|
