1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
#!/usr/bin/env python3
#
# Process img-bench test templates
#
# Copyright (c) 2021 Virtuozzo International GmbH.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import sys
import subprocess
import re
import json
import simplebench
from results_to_text import results_to_text
from table_templater import Templater
def bench_func(env, case):
test = templater.gen(env['data'], case['data'])
p = subprocess.run(test, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True)
if p.returncode == 0:
try:
m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout)
return {'seconds': float(m.group(1))}
except Exception:
return {'error': f'failed to parse qemu-img output: {p.stdout}'}
else:
return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'}
if __name__ == '__main__':
if len(sys.argv) > 1:
print("""
Usage: img_bench_templater.py < path/to/test-template.sh
This script generates performance tests from a test template (example below),
runs them, and displays the results in a table. The template is read from
stdin. It must be written in bash and end with a `qemu-img bench` invocation
(whose result is parsed to get the test instance’s result).
Use the following syntax in the template to create the various different test
instances:
column templating: {var1|var2|...} - test will use different values in
different columns. You may use several {} constructions in the test, in this
case product of all choice-sets will be used.
row templating: [var1|var2|...] - similar thing to define rows (test-cases)
Test template example:
Assume you want to compare two qemu-img binaries, called qemu-img-old and
qemu-img-new in your build directory in two test-cases with 4K writes and 64K
writes. The template may look like this:
qemu_img=/path/to/qemu/build/qemu-img-{old|new}
$qemu_img create -f qcow2 /ssd/x.qcow2 1G
$qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2
When passing this to stdin of img_bench_templater.py, the resulting comparison
table will contain two columns (for two binaries) and two rows (for two
test-cases).
In addition to displaying the results, script also stores results in JSON
format into results.json file in current directory.
""")
sys.exit()
templater = Templater(sys.stdin.read())
envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns]
cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows]
result = simplebench.bench(bench_func, envs, cases, count=5,
initial_run=False)
print(results_to_text(result))
with open('results.json', 'w') as f:
json.dump(result, f, indent=4)
|
