diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/coccinelle/cpu-reset.cocci | 47 | ||||
| -rw-r--r-- | scripts/coccinelle/memory-region-housekeeping.cocci | 159 | ||||
| -rw-r--r-- | scripts/coccinelle/memory-region-init-ram.cocci | 38 | ||||
| -rw-r--r-- | scripts/hxtool | 78 | ||||
| -rw-r--r-- | scripts/qapi/commands.py | 6 | ||||
| -rw-r--r-- | scripts/qapi/doc.py | 16 | ||||
| -rw-r--r-- | scripts/qapi/events.py | 2 | ||||
| -rw-r--r-- | scripts/qapi/expr.py | 14 | ||||
| -rw-r--r-- | scripts/qapi/introspect.py | 106 | ||||
| -rw-r--r-- | scripts/qapi/schema.py | 460 | ||||
| -rw-r--r-- | scripts/qapi/types.py | 8 | ||||
| -rw-r--r-- | scripts/qapi/visit.py | 8 | ||||
| -rw-r--r-- | scripts/simplebench/bench-example.py | 80 | ||||
| -rwxr-xr-x | scripts/simplebench/bench_block_job.py | 119 | ||||
| -rw-r--r-- | scripts/simplebench/simplebench.py | 128 |
15 files changed, 863 insertions, 406 deletions
diff --git a/scripts/coccinelle/cpu-reset.cocci b/scripts/coccinelle/cpu-reset.cocci new file mode 100644 index 0000000000..396a724e51 --- /dev/null +++ b/scripts/coccinelle/cpu-reset.cocci @@ -0,0 +1,47 @@ +// Convert targets using the old CPUState reset to DeviceState reset +// +// Copyright Linaro Ltd 2020 +// This work is licensed under the terms of the GNU GPLv2 or later. +// +// spatch --macro-file scripts/cocci-macro-file.h \ +// --sp-file scripts/coccinelle/cpu-reset.cocci \ +// --keep-comments --smpl-spacing --in-place --include-headers --dir target +// +// For simplicity we assume some things about the code we're modifying +// that happen to be true for all our targets: +// * all cpu_class_set_parent_reset() callsites have a 'DeviceClass *dc' local +// * the parent reset field in the target CPU class is 'parent_reset' +// * no reset function already has a 'dev' local + +@@ +identifier cpu, x; +typedef CPUState; +@@ +struct x { +... +- void (*parent_reset)(CPUState *cpu); ++ DeviceReset parent_reset; +... +}; +@ rule1 @ +identifier resetfn; +expression resetfield; +identifier cc; +@@ +- cpu_class_set_parent_reset(cc, resetfn, resetfield) ++ device_class_set_parent_reset(dc, resetfn, resetfield) +@@ +identifier rule1.resetfn; +identifier cpu, cc; +typedef CPUState, DeviceState; +@@ +-resetfn(CPUState *cpu) +-{ ++resetfn(DeviceState *dev) ++{ ++ CPUState *cpu = CPU(dev); +<... +- cc->parent_reset(cpu); ++ cc->parent_reset(dev); +...> +} diff --git a/scripts/coccinelle/memory-region-housekeeping.cocci b/scripts/coccinelle/memory-region-housekeeping.cocci new file mode 100644 index 0000000000..c768d8140a --- /dev/null +++ b/scripts/coccinelle/memory-region-housekeeping.cocci @@ -0,0 +1,159 @@ +/* + Usage: + + spatch \ + --macro-file scripts/cocci-macro-file.h \ + --sp-file scripts/coccinelle/memory-region-housekeeping.cocci \ + --keep-comments \ + --in-place \ + --dir . + +*/ + + +// Replace memory_region_init_ram(readonly) by memory_region_init_rom() +@@ +expression E1, E2, E3, E4, E5; +symbol true; +@@ +( +- memory_region_init_ram(E1, E2, E3, E4, E5); ++ memory_region_init_rom(E1, E2, E3, E4, E5); + ... WHEN != E1 +- memory_region_set_readonly(E1, true); +| +- memory_region_init_ram_nomigrate(E1, E2, E3, E4, E5); ++ memory_region_init_rom_nomigrate(E1, E2, E3, E4, E5); + ... WHEN != E1 +- memory_region_set_readonly(E1, true); +) + + +@possible_memory_region_init_rom@ +expression E1, E2, E3, E4, E5; +position p; +@@ +( + memory_region_init_ram@p(E1, E2, E3, E4, E5); + ... + memory_region_set_readonly(E1, true); +| + memory_region_init_ram_nomigrate@p(E1, E2, E3, E4, E5); + ... + memory_region_set_readonly(E1, true); +) +@script:python@ +p << possible_memory_region_init_rom.p; +@@ +cocci.print_main("potential use of memory_region_init_rom*() in ", p) + + +// Do not call memory_region_set_readonly() on ROM alias +@@ +expression ROM, E1, E2, E3, E4; +expression ALIAS, E5, E6, E7, E8; +@@ +( + memory_region_init_rom(ROM, E1, E2, E3, E4); +| + memory_region_init_rom_nomigrate(ROM, E1, E2, E3, E4); +) + ... + memory_region_init_alias(ALIAS, E5, E6, ROM, E7, E8); +- memory_region_set_readonly(ALIAS, true); + + +// Replace by-hand memory_region_init_ram_nomigrate/vmstate_register_ram +// code sequences with use of the new memory_region_init_ram function. +// Similarly for the _rom and _rom_device functions. +// We don't try to replace sequences with a non-NULL owner, because +// there are none in the tree that can be automatically converted +// (and only a handful that can be manually converted). +@@ +expression MR; +expression NAME; +expression SIZE; +expression ERRP; +@@ +-memory_region_init_ram_nomigrate(MR, NULL, NAME, SIZE, ERRP); ++memory_region_init_ram(MR, NULL, NAME, SIZE, ERRP); + ... +-vmstate_register_ram_global(MR); +@@ +expression MR; +expression NAME; +expression SIZE; +expression ERRP; +@@ +-memory_region_init_rom_nomigrate(MR, NULL, NAME, SIZE, ERRP); ++memory_region_init_rom(MR, NULL, NAME, SIZE, ERRP); + ... +-vmstate_register_ram_global(MR); +@@ +expression MR; +expression OPS; +expression OPAQUE; +expression NAME; +expression SIZE; +expression ERRP; +@@ +-memory_region_init_rom_device_nomigrate(MR, NULL, OPS, OPAQUE, NAME, SIZE, ERRP); ++memory_region_init_rom_device(MR, NULL, OPS, OPAQUE, NAME, SIZE, ERRP); + ... +-vmstate_register_ram_global(MR); + + +// Device is owner +@@ +typedef DeviceState; +identifier device_fn, dev, obj; +expression E1, E2, E3, E4, E5; +@@ +static void device_fn(DeviceState *dev, ...) +{ + ... + Object *obj = OBJECT(dev); + <+... +( +- memory_region_init(E1, NULL, E2, E3); ++ memory_region_init(E1, obj, E2, E3); +| +- memory_region_init_io(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_io(E1, obj, E2, E3, E4, E5); +| +- memory_region_init_alias(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_alias(E1, obj, E2, E3, E4, E5); +| +- memory_region_init_rom(E1, NULL, E2, E3, E4); ++ memory_region_init_rom(E1, obj, E2, E3, E4); +| +- memory_region_init_ram_shared_nomigrate(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_ram_shared_nomigrate(E1, obj, E2, E3, E4, E5); +) + ...+> +} +@@ +identifier device_fn, dev; +expression E1, E2, E3, E4, E5; +@@ +static void device_fn(DeviceState *dev, ...) +{ + <+... +( +- memory_region_init(E1, NULL, E2, E3); ++ memory_region_init(E1, OBJECT(dev), E2, E3); +| +- memory_region_init_io(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_io(E1, OBJECT(dev), E2, E3, E4, E5); +| +- memory_region_init_alias(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_alias(E1, OBJECT(dev), E2, E3, E4, E5); +| +- memory_region_init_rom(E1, NULL, E2, E3, E4); ++ memory_region_init_rom(E1, OBJECT(dev), E2, E3, E4); +| +- memory_region_init_ram_shared_nomigrate(E1, NULL, E2, E3, E4, E5); ++ memory_region_init_ram_shared_nomigrate(E1, OBJECT(dev), E2, E3, E4, E5); +) + ...+> +} diff --git a/scripts/coccinelle/memory-region-init-ram.cocci b/scripts/coccinelle/memory-region-init-ram.cocci deleted file mode 100644 index d290150872..0000000000 --- a/scripts/coccinelle/memory-region-init-ram.cocci +++ /dev/null @@ -1,38 +0,0 @@ -// Replace by-hand memory_region_init_ram_nomigrate/vmstate_register_ram -// code sequences with use of the new memory_region_init_ram function. -// Similarly for the _rom and _rom_device functions. -// We don't try to replace sequences with a non-NULL owner, because -// there are none in the tree that can be automatically converted -// (and only a handful that can be manually converted). -@@ -expression MR; -expression NAME; -expression SIZE; -expression ERRP; -@@ --memory_region_init_ram_nomigrate(MR, NULL, NAME, SIZE, ERRP); -+memory_region_init_ram(MR, NULL, NAME, SIZE, ERRP); - ... --vmstate_register_ram_global(MR); -@@ -expression MR; -expression NAME; -expression SIZE; -expression ERRP; -@@ --memory_region_init_rom_nomigrate(MR, NULL, NAME, SIZE, ERRP); -+memory_region_init_rom(MR, NULL, NAME, SIZE, ERRP); - ... --vmstate_register_ram_global(MR); -@@ -expression MR; -expression OPS; -expression OPAQUE; -expression NAME; -expression SIZE; -expression ERRP; -@@ --memory_region_init_rom_device_nomigrate(MR, NULL, OPS, OPAQUE, NAME, SIZE, ERRP); -+memory_region_init_rom_device(MR, NULL, OPS, OPAQUE, NAME, SIZE, ERRP); - ... --vmstate_register_ram_global(MR); diff --git a/scripts/hxtool b/scripts/hxtool index 0003e7b673..7b1452f3cf 100644 --- a/scripts/hxtool +++ b/scripts/hxtool @@ -7,7 +7,7 @@ hxtoh() case $str in HXCOMM*) ;; - STEXI*|ETEXI*|SRST*|ERST*) flag=$(($flag^1)) + SRST*|ERST*) flag=$(($flag^1)) ;; *) test $flag -eq 1 && printf "%s\n" "$str" @@ -16,84 +16,8 @@ hxtoh() done } -print_texi_heading() -{ - if test "$*" != ""; then - title="$*" - printf "@subsection %s\n" "${title%:}" - fi -} - -hxtotexi() -{ - flag=0 - rstflag=0 - line=1 - while read -r str; do - case "$str" in - HXCOMM*) - ;; - STEXI*) - if test $rstflag -eq 1 ; then - printf "line %d: syntax error: expected ERST, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - if test $flag -eq 1 ; then - printf "line %d: syntax error: expected ETEXI, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - flag=1 - ;; - ETEXI*) - if test $rstflag -eq 1 ; then - printf "line %d: syntax error: expected ERST, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - if test $flag -ne 1 ; then - printf "line %d: syntax error: expected STEXI, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - flag=0 - ;; - SRST*) - if test $rstflag -eq 1 ; then - printf "line %d: syntax error: expected ERST, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - if test $flag -eq 1 ; then - printf "line %d: syntax error: expected ETEXI, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - rstflag=1 - ;; - ERST*) - if test $flag -eq 1 ; then - printf "line %d: syntax error: expected ETEXI, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - if test $rstflag -ne 1 ; then - printf "line %d: syntax error: expected SRST, found '%s'\n" "$line" "$str" >&2 - exit 1 - fi - rstflag=0 - ;; - DEFHEADING*) - print_texi_heading "$(expr "$str" : "DEFHEADING(\(.*\))")" - ;; - ARCHHEADING*) - print_texi_heading "$(expr "$str" : "ARCHHEADING(\(.*\),.*)")" - ;; - *) - test $flag -eq 1 && printf '%s\n' "$str" - ;; - esac - line=$((line+1)) - done -} - case "$1" in "-h") hxtoh ;; -"-t") hxtotexi ;; *) exit 1 ;; esac diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index 0e13e82989..bc30876c88 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -283,9 +283,9 @@ void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds); prefix=self._prefix)) self._genc.add(gen_registry(self._regy.get_content(), self._prefix)) - def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, - success_response, boxed, allow_oob, allow_preconfig, - features): + def visit_command(self, name, info, ifcond, features, + arg_type, ret_type, gen, success_response, boxed, + allow_oob, allow_preconfig): if not gen: return # FIXME: If T is a user-defined type, the user is responsible diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py index 1787a53d91..92f584edcf 100644 --- a/scripts/qapi/doc.py +++ b/scripts/qapi/doc.py @@ -243,34 +243,34 @@ class QAPISchemaGenDocVisitor(QAPISchemaVisitor): def write(self, output_dir): self._gen.write(output_dir) - def visit_enum_type(self, name, info, ifcond, members, prefix): + def visit_enum_type(self, name, info, ifcond, features, members, prefix): doc = self.cur_doc self._gen.add(texi_type('Enum', doc, ifcond, texi_members(doc, 'Values', member_func=texi_enum_value))) - def visit_object_type(self, name, info, ifcond, base, members, variants, - features): + def visit_object_type(self, name, info, ifcond, features, + base, members, variants): doc = self.cur_doc if base and base.is_implicit(): base = None self._gen.add(texi_type('Object', doc, ifcond, texi_members(doc, 'Members', base, variants))) - def visit_alternate_type(self, name, info, ifcond, variants): + def visit_alternate_type(self, name, info, ifcond, features, variants): doc = self.cur_doc self._gen.add(texi_type('Alternate', doc, ifcond, texi_members(doc, 'Members'))) - def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, - success_response, boxed, allow_oob, allow_preconfig, - features): + def visit_command(self, name, info, ifcond, features, + arg_type, ret_type, gen, success_response, boxed, + allow_oob, allow_preconfig): doc = self.cur_doc self._gen.add(texi_msg('Command', doc, ifcond, texi_arguments(doc, arg_type if boxed else None))) - def visit_event(self, name, info, ifcond, arg_type, boxed): + def visit_event(self, name, info, ifcond, features, arg_type, boxed): doc = self.cur_doc self._gen.add(texi_msg('Event', doc, ifcond, texi_arguments(doc, diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py index a98b9f5099..b544af5a1c 100644 --- a/scripts/qapi/events.py +++ b/scripts/qapi/events.py @@ -189,7 +189,7 @@ void %(event_emit)s(%(event_enum)s event, QDict *qdict); event_emit=self._event_emit_name, event_enum=self._event_enum_name)) - def visit_event(self, name, info, ifcond, arg_type, boxed): + def visit_event(self, name, info, ifcond, features, arg_type, boxed): with ifcontext(ifcond, self._genh, self._genc): self._genh.add(gen_event_send_decl(name, arg_type, boxed)) self._genc.add(gen_event_send(name, arg_type, boxed, diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index fecf466fa7..2942520399 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -167,8 +167,9 @@ def check_type(value, info, source, allow_optional=True, permit_upper=permit_upper) if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): raise QAPISemError(info, "%s uses reserved name" % key_source) - check_keys(arg, info, key_source, ['type'], ['if']) + check_keys(arg, info, key_source, ['type'], ['if', 'features']) check_if(arg, info, key_source) + check_features(arg.get('features'), info) check_type(arg['type'], info, key_source, allow_array=True) @@ -219,7 +220,6 @@ def check_struct(expr, info): check_type(members, info, "'data'", allow_dict=name) check_type(expr.get('base'), info, "'base'") - check_features(expr.get('features'), info) def check_union(expr, info): @@ -267,7 +267,6 @@ def check_command(expr, info): raise QAPISemError(info, "'boxed': true requires 'data'") check_type(args, info, "'data'", allow_dict=not boxed) check_type(rets, info, "'returns'", allow_array=True) - check_features(expr.get('features'), info) def check_event(expr, info): @@ -319,18 +318,18 @@ def check_exprs(exprs): if meta == 'enum': check_keys(expr, info, meta, - ['enum', 'data'], ['if', 'prefix']) + ['enum', 'data'], ['if', 'features', 'prefix']) check_enum(expr, info) elif meta == 'union': check_keys(expr, info, meta, ['union', 'data'], - ['base', 'discriminator', 'if']) + ['base', 'discriminator', 'if', 'features']) normalize_members(expr.get('base')) normalize_members(expr['data']) check_union(expr, info) elif meta == 'alternate': check_keys(expr, info, meta, - ['alternate', 'data'], ['if']) + ['alternate', 'data'], ['if', 'features']) normalize_members(expr['data']) check_alternate(expr, info) elif meta == 'struct': @@ -348,13 +347,14 @@ def check_exprs(exprs): check_command(expr, info) elif meta == 'event': check_keys(expr, info, meta, - ['event'], ['data', 'boxed', 'if']) + ['event'], ['data', 'boxed', 'if', 'features']) normalize_members(expr.get('data')) check_event(expr, info) else: assert False, 'unexpected meta type' check_if(expr, info, meta) + check_features(expr.get('features'), info) check_flags(expr, info) return exprs diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index b5537eddc0..23652be810 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -16,7 +16,19 @@ from qapi.schema import (QAPISchemaArrayType, QAPISchemaBuiltinType, QAPISchemaType) -def to_qlit(obj, level=0, suppress_first_indent=False): +def _make_tree(obj, ifcond, features, extra=None): + if extra is None: + extra = {} + if ifcond: + extra['if'] = ifcond + if features: + obj['features'] = [(f.name, {'if': f.ifcond}) for f in features] + if extra: + return (obj, extra) + return obj + + +def _tree_to_qlit(obj, level=0, suppress_first_indent=False): def indent(level): return level * 4 * ' ' @@ -30,7 +42,7 @@ def to_qlit(obj, level=0, suppress_first_indent=False): ret += indent(level) + '/* %s */\n' % comment if ifcond: ret += gen_if(ifcond) - ret += to_qlit(ifobj, level) + ret += _tree_to_qlit(ifobj, level) if ifcond: ret += '\n' + gen_endif(ifcond) return ret @@ -43,7 +55,7 @@ def to_qlit(obj, level=0, suppress_first_indent=False): elif isinstance(obj, str): ret += 'QLIT_QSTR(' + to_c_string(obj) + ')' elif isinstance(obj, list): - elts = [to_qlit(elt, level + 1).strip('\n') + elts = [_tree_to_qlit(elt, level + 1).strip('\n') for elt in obj] elts.append(indent(level + 1) + "{}") ret += 'QLIT_QLIST(((QLitObject[]) {\n' @@ -53,7 +65,8 @@ def to_qlit(obj, level=0, suppress_first_indent=False): elts = [] for key, value in sorted(obj.items()): elts.append(indent(level + 1) + '{ %s, %s }' % - (to_c_string(key), to_qlit(value, level + 1, True))) + (to_c_string(key), + _tree_to_qlit(value, level + 1, True))) elts.append(indent(level + 1) + '{}') ret += 'QLIT_QDICT(((QLitDictEntry[]) {\n' ret += ',\n'.join(elts) + '\n' @@ -79,7 +92,7 @@ class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor): ' * QAPI/QMP schema introspection', __doc__) self._unmask = unmask self._schema = None - self._qlits = [] + self._trees = [] self._used_types = [] self._name_map = {} self._genc.add(mcgen(''' @@ -108,9 +121,9 @@ extern const QLitObject %(c_name)s; const QLitObject %(c_name)s = %(c_string)s; ''', c_name=c_name(name), - c_string=to_qlit(self._qlits))) + c_string=_tree_to_qlit(self._trees))) self._schema = None - self._qlits = [] + self._trees = [] self._used_types = [] self._name_map = {} @@ -144,89 +157,78 @@ const QLitObject %(c_name)s = %(c_string)s; return '[' + self._use_type(typ.element_type) + ']' return self._name(typ.name) - def _gen_qlit(self, name, mtype, obj, ifcond): - extra = {} + def _gen_tree(self, name, mtype, obj, ifcond, features): + extra = None if mtype not in ('command', 'event', 'builtin', 'array'): if not self._unmask: # Output a comment to make it easy to map masked names # back to the source when reading the generated output. - extra['comment'] = '"%s" = %s' % (self._name(name), name) + extra = {'comment': '"%s" = %s' % (self._name(name), name)} name = self._name(name) obj['name'] = name obj['meta-type'] = mtype - if ifcond: - extra['if'] = ifcond - if extra: - self._qlits.append((obj, extra)) - else: - self._qlits.append(obj) + self._trees.append(_make_tree(obj, ifcond, features, extra)) def _gen_member(self, member): - ret = {'name': member.name, 'type': self._use_type(member.type)} + obj = {'name': member.name, 'type': self._use_type(member.type)} if member.optional: - ret['default'] = None - if member.ifcond: - ret = (ret, {'if': member.ifcond}) - return ret + obj['default'] = None + return _make_tree(obj, member.ifcond, member.features) def _gen_variants(self, tag_name, variants): return {'tag': tag_name, 'variants': [self._gen_variant(v) for v in variants]} def _gen_variant(self, variant): - return ({'case': variant.name, 'type': self._use_type(variant.type)}, - {'if': variant.ifcond}) + obj = {'case': variant.name, 'type': self._use_type(variant.type)} + return _make_tree(obj, variant.ifcond, None) def visit_builtin_type(self, name, info, json_type): - self._gen_qlit(name, 'builtin', {'json-type': json_type}, []) + self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None) - def visit_enum_type(self, name, info, ifcond, members, prefix): - self._gen_qlit(name, 'enum', - {'values': - [(m.name, {'if': m.ifcond}) for m in members]}, - ifcond) + def visit_enum_type(self, name, info, ifcond, features, members, prefix): + self._gen_tree(name, 'enum', + {'values': [_make_tree(m.name, m.ifcond, None) + for m in members]}, + ifcond, features) def visit_array_type(self, name, info, ifcond, element_type): element = self._use_type(element_type) - self._gen_qlit('[' + element + ']', 'array', {'element-type': element}, - ifcond) + self._gen_tree('[' + element + ']', 'array', {'element-type': element}, + ifcond, None) - def visit_object_type_flat(self, name, info, ifcond, members, variants, - features): + def visit_object_type_flat(self, name, info, ifcond, features, + members, variants): obj = {'members': [self._gen_member(m) for m in members]} if variants: obj.update(self._gen_variants(variants.tag_member.name, variants.variants)) - if features: - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features] - self._gen_qlit(name, 'object', obj, ifcond) + self._gen_tree(name, 'object', obj, ifcond, features) - def visit_alternate_type(self, name, info, ifcond, variants): - self._gen_qlit(name, 'alternate', + def visit_alternate_type(self, name, info, ifcond, features, variants): + self._gen_tree(name, 'alternate', {'members': [ - ({'type': self._use_type(m.type)}, {'if': m.ifcond}) - for m in variants.variants]}, ifcond) - - def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, - success_response, boxed, allow_oob, allow_preconfig, - features): + _make_tree({'type': self._use_type(m.type)}, + m.ifcond, None) + for m in variants.variants]}, + ifcond, features) + + def visit_command(self, name, info, ifcond, features, + arg_type, ret_type, gen, success_response, boxed, + allow_oob, allow_preconfig): arg_type = arg_type or self._schema.the_empty_object_type ret_type = ret_type or self._schema.the_empty_object_type obj = {'arg-type': self._use_type(arg_type), 'ret-type': self._use_type(ret_type)} if allow_oob: obj['allow-oob'] = allow_oob + self._gen_tree(name, 'command', obj, ifcond, features) - if features: - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features] - - self._gen_qlit(name, 'command', obj, ifcond) - - def visit_event(self, name, info, ifcond, arg_type, boxed): + def visit_event(self, name, info, ifcond, features, arg_type, boxed): arg_type = arg_type or self._schema.the_empty_object_type - self._gen_qlit(name, 'event', {'arg-type': self._use_type(arg_type)}, - ifcond) + self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)}, + ifcond, features) def gen_introspect(schema, output_dir, prefix, opt_unmask): diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index d759308b4e..78309a00f0 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -53,13 +53,13 @@ class QAPISchemaEntity: seen = {} for f in self.features: f.check_clash(self.info, seen) - if self.doc: - self.doc.connect_feature(f) - self._checked = True def connect_doc(self, doc=None): - pass + doc = doc or self.doc + if doc: + for f in self.features: + doc.connect_feature(f) def check_doc(self): if self.doc: @@ -109,29 +109,29 @@ class QAPISchemaVisitor: def visit_builtin_type(self, name, info, json_type): pass - def visit_enum_type(self, name, info, ifcond, members, prefix): + def visit_enum_type(self, name, info, ifcond, features, members, prefix): pass def visit_array_type(self, name, info, ifcond, element_type): pass - def visit_object_type(self, name, info, ifcond, base, members, variants, - features): + def visit_object_type(self, name, info, ifcond, features, + base, members, variants): pass - def visit_object_type_flat(self, name, info, ifcond, members, variants, - features): + def visit_object_type_flat(self, name, info, ifcond, features, + members, variants): pass - def visit_alternate_type(self, name, info, ifcond, variants): + def visit_alternate_type(self, name, info, ifcond, features, variants): pass - def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, - success_response, boxed, allow_oob, allow_preconfig, - features): + def visit_command(self, name, info, ifcond, features, + arg_type, ret_type, gen, success_response, boxed, + allow_oob, allow_preconfig): pass - def visit_event(self, name, info, ifcond, arg_type, boxed): + def visit_event(self, name, info, ifcond, features, arg_type, boxed): pass @@ -193,6 +193,12 @@ class QAPISchemaType(QAPISchemaEntity): return None return self.name + def check(self, schema): + QAPISchemaEntity.check(self, schema) + if 'deprecated' in [f.name for f in self.features]: + raise QAPISemError( + self.info, "feature 'deprecated' is not supported for types") + def describe(self): assert self.meta return "%s type '%s'" % (self.meta, self.name) @@ -234,8 +240,8 @@ class QAPISchemaBuiltinType(QAPISchemaType): class QAPISchemaEnumType(QAPISchemaType): meta = 'enum' - def __init__(self, name, info, doc, ifcond, members, prefix): - super().__init__(name, info, doc, ifcond) + def __init__(self, name, info, doc, ifcond, features, members, prefix): + super().__init__(name, info, doc, ifcond, features) for m in members: assert isinstance(m, QAPISchemaEnumMember) m.set_defined_in(name) @@ -250,10 +256,10 @@ class QAPISchemaEnumType(QAPISchemaType): m.check_clash(self.info, seen) def connect_doc(self, doc=None): + super().connect_doc(doc) doc = doc or self.doc - if doc: - for m in self.members: - doc.connect_member(m) + for m in self.members: + m.connect_doc(doc) def is_implicit(self): # See QAPISchema._make_implicit_enum_type() and ._def_predefineds() @@ -270,15 +276,16 @@ class QAPISchemaEnumType(QAPISchemaType): def visit(self, visitor): super().visit(visitor) - visitor.visit_enum_type(self.name, self.info, self.ifcond, - self.members, self.prefix) + visitor.visit_enum_type( + self.name, self.info, self.ifcond, self.features, + self.members, self.prefix) class QAPISchemaArrayType(QAPISchemaType): meta = 'array' def __init__(self, name, info, element_type): - super().__init__(name, info, None, None) + super().__init__(name, info, None) assert isinstance(element_type, str) self._element_type_name = element_type self.element_type = None @@ -324,8 +331,8 @@ class QAPISchemaArrayType(QAPISchemaType): class QAPISchemaObjectType(QAPISchemaType): - def __init__(self, name, info, doc, ifcond, - base, local_members, variants, features): + def __init__(self, name, info, doc, ifcond, features, + base, local_members, variants): # struct has local_members, optional base, and no variants # flat union has base, variants, and no local_members # simple union has local_members, variants, and no base @@ -336,7 +343,7 @@ class QAPISchemaObjectType(QAPISchemaType): assert isinstance(m, QAPISchemaObjectTypeMember) m.set_defined_in(name) if variants is not None: - assert isinstance(variants, QAPISchemaObjectTypeVariants) + assert isinstance(variants, QAPISchemaVariants) variants.set_defined_in(name) self._base_name = base self.base = None @@ -392,12 +399,12 @@ class QAPISchemaObjectType(QAPISchemaType): m.check_clash(info, seen) def connect_doc(self, doc=None): + super().connect_doc(doc) doc = doc or self.doc - if doc: - if self.base and self.base.is_implicit(): - self.base.connect_doc(doc) - for m in self.local_members: - doc.connect_member(m) + if self.base and self.base.is_implicit(): + self.base.connect_doc(doc) + for m in self.local_members: + m.connect_doc(doc) @property def ifcond(self): @@ -433,93 +440,82 @@ class QAPISchemaObjectType(QAPISchemaType): def visit(self, visitor): super().visit(visitor) - visitor.visit_object_type(self.name, self.info, self.ifcond, - self.base, self.local_members, self.variants, - self.features) - visitor.visit_object_type_flat(self.name, self.info, self.ifcond, - self.members, self.variants, - self.features) - - -class QAPISchemaMember: - """ Represents object members, enum members and features """ - role = 'member' - - def __init__(self, name, info, ifcond=None): - assert isinstance(name, str) - self.name = name - self.info = info - self.ifcond = ifcond or [] - self.defined_in = None - - def set_defined_in(self, name): - assert not self.defined_in - self.defined_in = name - - def check_clash(self, info, seen): - cname = c_name(self.name) - if cname in seen: - raise QAPISemError( - info, - "%s collides with %s" - % (self.describe(info), seen[cname].describe(info))) - seen[cname] = self + visitor.visit_object_type( + self.name, self.info, self.ifcond, self.features, + self.base, self.local_members, self.variants) + visitor.visit_object_type_flat( + self.name, self.info, self.ifcond, self.features, + self.members, self.variants) - def describe(self, info): - role = self.role - defined_in = self.defined_in - assert defined_in - - if defined_in.startswith('q_obj_'): - # See QAPISchema._make_implicit_object_type() - reverse the - # mapping there to create a nice human-readable description - defined_in = defined_in[6:] - if defined_in.endswith('-arg'): - # Implicit type created for a command's dict 'data' - assert role == 'member' - role = 'parameter' - elif defined_in.endswith('-base'): - # Implicit type created for a flat union's dict 'base' - role = 'base ' + role - else: - # Implicit type created for a simple union's branch - assert defined_in.endswith('-wrapper') - # Unreachable and not implemented - assert False - elif defined_in.endswith('Kind'): - # See QAPISchema._make_implicit_enum_type() - # Implicit enum created for simple union's branches - assert role == 'value' - role = 'branch' - elif defined_in != info.defn_name: - return "%s '%s' of type '%s'" % (role, self.name, defined_in) - return "%s '%s'" % (role, self.name) +class QAPISchemaAlternateType(QAPISchemaType): + meta = 'alternate' -class QAPISchemaEnumMember(QAPISchemaMember): - role = 'value' + def __init__(self, name, info, doc, ifcond, features, variants): + super().__init__(name, info, doc, ifcond, features) + assert isinstance(variants, QAPISchemaVariants) + assert variants.tag_member + variants.set_defined_in(name) + variants.tag_member.set_defined_in(self.name) + self.variants = variants + def check(self, schema): + super().check(schema) + self.variants.tag_member.check(schema) + # Not calling self.variants.check_clash(), because there's nothing + # to clash with + self.variants.check(schema, {}) + # Alternate branch names have no relation to the tag enum values; + # so we have to check for potential name collisions ourselves. + seen = {} + types_seen = {} + for v in self.variants.variants: + v.check_clash(self.info, seen) + qtype = v.type.alternate_qtype() + if not qtype: + raise QAPISemError( + self.info, + "%s cannot use %s" + % (v.describe(self.info), v.type.describe())) + conflicting = set([qtype]) + if qtype == 'QTYPE_QSTRING': + if isinstance(v.type, QAPISchemaEnumType): + for m in v.type.members: + if m.name in ['on', 'off']: + conflicting.add('QTYPE_QBOOL') + if re.match(r'[-+0-9.]', m.name): + # lazy, could be tightened + conflicting.add('QTYPE_QNUM') + else: + conflicting.add('QTYPE_QNUM') + conflicting.add('QTYPE_QBOOL') + for qt in conflicting: + if qt in types_seen: + raise QAPISemError( + self.info, + "%s can't be distinguished from '%s'" + % (v.describe(self.info), types_seen[qt])) + types_seen[qt] = v.name -class QAPISchemaFeature(QAPISchemaMember): - role = 'feature' + def connect_doc(self, doc=None): + super().connect_doc(doc) + doc = doc or self.doc + for v in self.variants.variants: + v.connect_doc(doc) + def c_type(self): + return c_name(self.name) + pointer_suffix -class QAPISchemaObjectTypeMember(QAPISchemaMember): - def __init__(self, name, info, typ, optional, ifcond=None): - super().__init__(name, info, ifcond) - assert isinstance(typ, str) - assert isinstance(optional, bool) - self._type_name = typ - self.type = None - self.optional = optional + def json_type(self): + return 'value' - def check(self, schema): - assert self.defined_in - self.type = schema.resolve_type(self._type_name, self.info, - self.describe) + def visit(self, visitor): + super().visit(visitor) + visitor.visit_alternate_type( + self.name, self.info, self.ifcond, self.features, self.variants) -class QAPISchemaObjectTypeVariants: +class QAPISchemaVariants: def __init__(self, tag_name, info, tag_member, variants): # Flat unions pass tag_name but not tag_member. # Simple unions and alternates pass tag_member but not tag_name. @@ -529,7 +525,7 @@ class QAPISchemaObjectTypeVariants: assert (isinstance(tag_name, str) or isinstance(tag_member, QAPISchemaObjectTypeMember)) for v in variants: - assert isinstance(v, QAPISchemaObjectTypeVariant) + assert isinstance(v, QAPISchemaVariant) self._tag_name = tag_name self.info = info self.tag_member = tag_member @@ -579,8 +575,8 @@ class QAPISchemaObjectTypeVariants: cases = {v.name for v in self.variants} for m in self.tag_member.type.members: if m.name not in cases: - v = QAPISchemaObjectTypeVariant(m.name, self.info, - 'q_empty', m.ifcond) + v = QAPISchemaVariant(m.name, self.info, + 'q_empty', m.ifcond) v.set_defined_in(self.tag_member.defined_in) self.variants.append(v) if not self.variants: @@ -610,86 +606,114 @@ class QAPISchemaObjectTypeVariants: v.type.check_clash(info, dict(seen)) -class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember): - role = 'branch' +class QAPISchemaMember: + """ Represents object members, enum members and features """ + role = 'member' - def __init__(self, name, info, typ, ifcond=None): - super().__init__(name, info, typ, False, ifcond) + def __init__(self, name, info, ifcond=None): + assert isinstance(name, str) + self.name = name + self.info = info + self.ifcond = ifcond or [] + self.defined_in = None + def set_defined_in(self, name): + assert not self.defined_in + self.defined_in = name -class QAPISchemaAlternateType(QAPISchemaType): - meta = 'alternate' + def check_clash(self, info, seen): + cname = c_name(self.name) + if cname in seen: + raise QAPISemError( + info, + "%s collides with %s" + % (self.describe(info), seen[cname].describe(info))) + seen[cname] = self + + def connect_doc(self, doc): + if doc: + doc.connect_member(self) + + def describe(self, info): + role = self.role + defined_in = self.defined_in + assert defined_in + + if defined_in.startswith('q_obj_'): + # See QAPISchema._make_implicit_object_type() - reverse the + # mapping there to create a nice human-readable description + defined_in = defined_in[6:] + if defined_in.endswith('-arg'): + # Implicit type created for a command's dict 'data' + assert role == 'member' + role = 'parameter' + elif defined_in.endswith('-base'): + # Implicit type created for a flat union's dict 'base' + role = 'base ' + role + else: + # Implicit type created for a simple union's branch + assert defined_in.endswith('-wrapper') + # Unreachable and not implemented + assert False + elif defined_in.endswith('Kind'): + # See QAPISchema._make_implicit_enum_type() + # Implicit enum created for simple union's branches + assert role == 'value' + role = 'branch' + elif defined_in != info.defn_name: + return "%s '%s' of type '%s'" % (role, self.name, defined_in) + return "%s '%s'" % (role, self.name) + + +class QAPISchemaEnumMember(QAPISchemaMember): + role = 'value' + + +class QAPISchemaFeature(QAPISchemaMember): + role = 'feature' - def __init__(self, name, info, doc, ifcond, variants): - super().__init__(name, info, doc, ifcond) - assert isinstance(variants, QAPISchemaObjectTypeVariants) - assert variants.tag_member - variants.set_defined_in(name) - variants.tag_member.set_defined_in(self.name) - self.variants = variants + +class QAPISchemaObjectTypeMember(QAPISchemaMember): + def __init__(self, name, info, typ, optional, ifcond=None, features=None): + super().__init__(name, info, ifcond) + assert isinstance(typ, str) + assert isinstance(optional, bool) + for f in features or []: + assert isinstance(f, QAPISchemaFeature) + f.set_defined_in(name) + self._type_name = typ + self.type = None + self.optional = optional + self.features = features or [] def check(self, schema): - super().check(schema) - self.variants.tag_member.check(schema) - # Not calling self.variants.check_clash(), because there's nothing - # to clash with - self.variants.check(schema, {}) - # Alternate branch names have no relation to the tag enum values; - # so we have to check for potential name collisions ourselves. + assert self.defined_in + self.type = schema.resolve_type(self._type_name, self.info, + self.describe) seen = {} - types_seen = {} - for v in self.variants.variants: - v.check_clash(self.info, seen) - qtype = v.type.alternate_qtype() - if not qtype: - raise QAPISemError( - self.info, - "%s cannot use %s" - % (v.describe(self.info), v.type.describe())) - conflicting = set([qtype]) - if qtype == 'QTYPE_QSTRING': - if isinstance(v.type, QAPISchemaEnumType): - for m in v.type.members: - if m.name in ['on', 'off']: - conflicting.add('QTYPE_QBOOL') - if re.match(r'[-+0-9.]', m.name): - # lazy, could be tightened - conflicting.add('QTYPE_QNUM') - else: - conflicting.add('QTYPE_QNUM') - conflicting.add('QTYPE_QBOOL') - for qt in conflicting: - if qt in types_seen: - raise QAPISemError( - self.info, - "%s can't be distinguished from '%s'" - % (v.describe(self.info), types_seen[qt])) - types_seen[qt] = v.name + for f in self.features: + f.check_clash(self.info, seen) - def connect_doc(self, doc=None): - doc = doc or self.doc + def connect_doc(self, doc): + super().connect_doc(doc) if doc: - for v in self.variants.variants: - doc.connect_member(v) + for f in self.features: + doc.connect_feature(f) - def c_type(self): - return c_name(self.name) + pointer_suffix - def json_type(self): - return 'value' +class QAPISchemaVariant(QAPISchemaObjectTypeMember): + role = 'branch' - def visit(self, visitor): - super().visit(visitor) - visitor.visit_alternate_type(self.name, self.info, self.ifcond, - self.variants) + def __init__(self, name, info, typ, ifcond=None): + super().__init__(name, info, typ, False, ifcond) class QAPISchemaCommand(QAPISchemaEntity): meta = 'command' - def __init__(self, name, info, doc, ifcond, arg_type, ret_type, - gen, success_response, boxed, allow_oob, allow_preconfig, - features): + def __init__(self, name, info, doc, ifcond, features, + arg_type, ret_type, + gen, success_response, boxed, allow_oob, allow_preconfig): super().__init__(name, info, doc, ifcond, features) assert not arg_type or isinstance(arg_type, str) assert not ret_type or isinstance(ret_type, str) @@ -733,6 +757,7 @@ class QAPISchemaCommand(QAPISchemaEntity): % self.ret_type.describe()) def connect_doc(self, doc=None): + super().connect_doc(doc) doc = doc or self.doc if doc: if self.arg_type and self.arg_type.is_implicit(): @@ -740,19 +765,17 @@ class QAPISchemaCommand(QAPISchemaEntity): def visit(self, visitor): super().visit(visitor) - visitor.visit_command(self.name, self.info, self.ifcond, - self.arg_type, self.ret_type, - self.gen, self.success_response, - self.boxed, self.allow_oob, - self.allow_preconfig, - self.features) + visitor.visit_command( + self.name, self.info, self.ifcond, self.features, + self.arg_type, self.ret_type, self.gen, self.success_response, + self.boxed, self.allow_oob, self.allow_preconfig) class QAPISchemaEvent(QAPISchemaEntity): meta = 'event' - def __init__(self, name, info, doc, ifcond, arg_type, boxed): - super().__init__(name, info, doc, ifcond) + def __init__(self, name, info, doc, ifcond, features, arg_type, boxed): + super().__init__(name, info, doc, ifcond, features) assert not arg_type or isinstance(arg_type, str) self._arg_type_name = arg_type self.arg_type = None @@ -775,6 +798,7 @@ class QAPISchemaEvent(QAPISchemaEntity): % self.arg_type.describe()) def connect_doc(self, doc=None): + super().connect_doc(doc) doc = doc or self.doc if doc: if self.arg_type and self.arg_type.is_implicit(): @@ -782,8 +806,9 @@ class QAPISchemaEvent(QAPISchemaEntity): def visit(self, visitor): super().visit(visitor) - visitor.visit_event(self.name, self.info, self.ifcond, - self.arg_type, self.boxed) + visitor.visit_event( + self.name, self.info, self.ifcond, self.features, + self.arg_type, self.boxed) class QAPISchema: @@ -888,7 +913,7 @@ class QAPISchema: ('null', 'null', 'QNull' + pointer_suffix)]: self._def_builtin_type(*t) self.the_empty_object_type = QAPISchemaObjectType( - 'q_empty', None, None, None, None, [], None, []) + 'q_empty', None, None, None, None, None, [], None) self._def_entity(self.the_empty_object_type) qtypes = ['none', 'qnull', 'qnum', 'qstring', 'qdict', 'qlist', @@ -896,10 +921,12 @@ class QAPISchema: qtype_values = self._make_enum_members( [{'name': n} for n in qtypes], None) - self._def_entity(QAPISchemaEnumType('QType', None, None, None, + self._def_entity(QAPISchemaEnumType('QType', None, None, None, None, qtype_values, 'QTYPE')) def _make_features(self, features, info): + if features is None: + return [] return [QAPISchemaFeature(f['name'], info, f.get('if')) for f in features] @@ -911,7 +938,8 @@ class QAPISchema: # See also QAPISchemaObjectTypeMember.describe() name = name + 'Kind' # reserved by check_defn_name_str() self._def_entity(QAPISchemaEnumType( - name, info, None, ifcond, self._make_enum_members(values, info), + name, info, None, ifcond, None, + self._make_enum_members(values, info), None)) return name @@ -939,8 +967,8 @@ class QAPISchema: # TODO kill simple unions or implement the disjunction assert (ifcond or []) == typ._ifcond # pylint: disable=protected-access else: - self._def_entity(QAPISchemaObjectType(name, info, None, ifcond, - None, members, None, [])) + self._def_entity(QAPISchemaObjectType( + name, info, None, ifcond, None, None, members, None)) return name def _def_enum_type(self, expr, info, doc): @@ -948,11 +976,12 @@ class QAPISchema: data = expr['data'] prefix = expr.get('prefix') ifcond = expr.get('if') + features = self._make_features(expr.get('features'), info) self._def_entity(QAPISchemaEnumType( - name, info, doc, ifcond, + name, info, doc, ifcond, features, self._make_enum_members(data, info), prefix)) - def _make_member(self, name, typ, ifcond, info): + def _make_member(self, name, typ, ifcond, features, info): optional = False if name.startswith('*'): name = name[1:] @@ -960,10 +989,12 @@ class QAPISchema: if isinstance(typ, list): assert len(typ) == 1 typ = self._make_array_type(typ[0], info) - return QAPISchemaObjectTypeMember(name, info, typ, optional, ifcond) + return QAPISchemaObjectTypeMember(name, info, typ, optional, ifcond, + self._make_features(features, info)) def _make_members(self, data, info): - return [self._make_member(key, value['type'], value.get('if'), info) + return [self._make_member(key, value['type'], value.get('if'), + value.get('features'), info) for (key, value) in data.items()] def _def_struct_type(self, expr, info, doc): @@ -971,15 +1002,14 @@ class QAPISchema: base = expr.get('base') data = expr['data'] ifcond = expr.get('if') - features = expr.get('features', []) + features = self._make_features(expr.get('features'), info) self._def_entity(QAPISchemaObjectType( - name, info, doc, ifcond, base, + name, info, doc, ifcond, features, base, self._make_members(data, info), - None, - self._make_features(features, info))) + None)) def _make_variant(self, case, typ, ifcond, info): - return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) + return QAPISchemaVariant(case, info, typ, ifcond) def _make_simple_variant(self, case, typ, ifcond, info): if isinstance(typ, list): @@ -987,14 +1017,15 @@ class QAPISchema: typ = self._make_array_type(typ[0], info) typ = self._make_implicit_object_type( typ, info, self.lookup_type(typ), - 'wrapper', [self._make_member('data', typ, None, info)]) - return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) + 'wrapper', [self._make_member('data', typ, None, None, info)]) + return QAPISchemaVariant(case, info, typ, ifcond) def _def_union_type(self, expr, info, doc): name = expr['union'] data = expr['data'] base = expr.get('base') ifcond = expr.get('if') + features = self._make_features(expr.get('features'), info) tag_name = expr.get('discriminator') tag_member = None if isinstance(base, dict): @@ -1015,22 +1046,23 @@ class QAPISchema: tag_member = QAPISchemaObjectTypeMember('type', info, typ, False) members = [tag_member] self._def_entity( - QAPISchemaObjectType(name, info, doc, ifcond, base, members, - QAPISchemaObjectTypeVariants( - tag_name, info, tag_member, variants), - [])) + QAPISchemaObjectType(name, info, doc, ifcond, features, + base, members, + QAPISchemaVariants( + tag_name, info, tag_member, variants))) def _def_alternate_type(self, expr, info, doc): name = expr['alternate'] data = expr['data'] ifcond = expr.get('if') + features = self._make_features(expr.get('features'), info) variants = [self._make_variant(key, value['type'], value.get('if'), info) for (key, value) in data.items()] tag_member = QAPISchemaObjectTypeMember('type', info, 'QType', False) self._def_entity( - QAPISchemaAlternateType(name, info, doc, ifcond, - QAPISchemaObjectTypeVariants( + QAPISchemaAlternateType(name, info, doc, ifcond, features, + QAPISchemaVariants( None, info, tag_member, variants))) def _def_command(self, expr, info, doc): @@ -1043,27 +1075,31 @@ class QAPISchema: allow_oob = expr.get('allow-oob', False) allow_preconfig = expr.get('allow-preconfig', False) ifcond = expr.get('if') - features = expr.get('features', []) + features = self._make_features(expr.get('features'), info) if isinstance(data, OrderedDict): data = self._make_implicit_object_type( - name, info, ifcond, 'arg', self._make_members(data, info)) + name, info, ifcond, + 'arg', self._make_members(data, info)) if isinstance(rets, list): assert len(rets) == 1 rets = self._make_array_type(rets[0], info) - self._def_entity(QAPISchemaCommand(name, info, doc, ifcond, data, rets, + self._def_entity(QAPISchemaCommand(name, info, doc, ifcond, features, + data, rets, gen, success_response, - boxed, allow_oob, allow_preconfig, - self._make_features(features, info))) + boxed, allow_oob, allow_preconfig)) def _def_event(self, expr, info, doc): name = expr['event'] data = expr.get('data') boxed = expr.get('boxed', False) ifcond = expr.get('if') + features = self._make_features(expr.get('features'), info) if isinstance(data, OrderedDict): data = self._make_implicit_object_type( - name, info, ifcond, 'arg', self._make_members(data, info)) - self._def_entity(QAPISchemaEvent(name, info, doc, ifcond, data, boxed)) + name, info, ifcond, + 'arg', self._make_members(data, info)) + self._def_entity(QAPISchemaEvent(name, info, doc, ifcond, features, + data, boxed)) def _def_exprs(self, exprs): for expr_elem in exprs: diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py index 3c83b6e4be..3ad33af4ee 100644 --- a/scripts/qapi/types.py +++ b/scripts/qapi/types.py @@ -278,7 +278,7 @@ class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor): self._genh.add(gen_type_cleanup_decl(name)) self._genc.add(gen_type_cleanup(name)) - def visit_enum_type(self, name, info, ifcond, members, prefix): + def visit_enum_type(self, name, info, ifcond, features, members, prefix): with ifcontext(ifcond, self._genh, self._genc): self._genh.preamble_add(gen_enum(name, members, prefix)) self._genc.add(gen_enum_lookup(name, members, prefix)) @@ -289,8 +289,8 @@ class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor): self._genh.add(gen_array(name, element_type)) self._gen_type_cleanup(name) - def visit_object_type(self, name, info, ifcond, base, members, variants, - features): + def visit_object_type(self, name, info, ifcond, features, + base, members, variants): # Nothing to do for the special empty builtin if name == 'q_empty': return @@ -306,7 +306,7 @@ class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor): # implicit types won't be directly allocated/freed self._gen_type_cleanup(name) - def visit_alternate_type(self, name, info, ifcond, variants): + def visit_alternate_type(self, name, info, ifcond, features, variants): with ifcontext(ifcond, self._genh): self._genh.preamble_add(gen_fwd_object_or_array(name)) self._genh.add(gen_object(name, ifcond, None, diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py index 421e5bd8cd..23d9194aa4 100644 --- a/scripts/qapi/visit.py +++ b/scripts/qapi/visit.py @@ -316,7 +316,7 @@ class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor): ''', types=types)) - def visit_enum_type(self, name, info, ifcond, members, prefix): + def visit_enum_type(self, name, info, ifcond, features, members, prefix): with ifcontext(ifcond, self._genh, self._genc): self._genh.add(gen_visit_decl(name, scalar=True)) self._genc.add(gen_visit_enum(name)) @@ -326,8 +326,8 @@ class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor): self._genh.add(gen_visit_decl(name)) self._genc.add(gen_visit_list(name, element_type)) - def visit_object_type(self, name, info, ifcond, base, members, variants, - features): + def visit_object_type(self, name, info, ifcond, features, + base, members, variants): # Nothing to do for the special empty builtin if name == 'q_empty': return @@ -342,7 +342,7 @@ class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor): self._genh.add(gen_visit_decl(name)) self._genc.add(gen_visit_object(name, base, members, variants)) - def visit_alternate_type(self, name, info, ifcond, variants): + def visit_alternate_type(self, name, info, ifcond, features, variants): with ifcontext(ifcond, self._genh, self._genc): self._genh.add(gen_visit_decl(name)) self._genc.add(gen_visit_alternate(name, variants)) diff --git a/scripts/simplebench/bench-example.py b/scripts/simplebench/bench-example.py new file mode 100644 index 0000000000..c642a5b891 --- /dev/null +++ b/scripts/simplebench/bench-example.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Benchmark example +# +# Copyright (c) 2019 Virtuozzo International GmbH. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import simplebench +from bench_block_job import bench_block_copy, drv_file, drv_nbd + + +def bench_func(env, case): + """ Handle one "cell" of benchmarking table. """ + return bench_block_copy(env['qemu_binary'], env['cmd'], + case['source'], case['target']) + + +# You may set the following five variables to correct values, to turn this +# example to real benchmark. +ssd_source = '/path-to-raw-source-image-at-ssd' +ssd_target = '/path-to-raw-target-image-at-ssd' +hdd_target = '/path-to-raw-source-image-at-hdd' +nbd_ip = 'nbd-ip-addr' +nbd_port = 'nbd-port-number' + +# Test-cases are "rows" in benchmark resulting table, 'id' is a caption for +# the row, other fields are handled by bench_func. +test_cases = [ + { + 'id': 'ssd -> ssd', + 'source': drv_file(ssd_source), + 'target': drv_file(ssd_target) + }, + { + 'id': 'ssd -> hdd', + 'source': drv_file(ssd_source), + 'target': drv_file(hdd_target) + }, + { + 'id': 'ssd -> nbd', + 'source': drv_file(ssd_source), + 'target': drv_nbd(nbd_ip, nbd_port) + }, +] + +# Test-envs are "columns" in benchmark resulting table, 'id is a caption for +# the column, other fields are handled by bench_func. +test_envs = [ + { + 'id': 'backup-1', + 'cmd': 'blockdev-backup', + 'qemu_binary': '/path-to-qemu-binary-1' + }, + { + 'id': 'backup-2', + 'cmd': 'blockdev-backup', + 'qemu_binary': '/path-to-qemu-binary-2' + }, + { + 'id': 'mirror', + 'cmd': 'blockdev-mirror', + 'qemu_binary': '/path-to-qemu-binary-1' + } +] + +result = simplebench.bench(bench_func, test_envs, test_cases, count=3) +print(simplebench.ascii(result)) diff --git a/scripts/simplebench/bench_block_job.py b/scripts/simplebench/bench_block_job.py new file mode 100755 index 0000000000..9808d696cf --- /dev/null +++ b/scripts/simplebench/bench_block_job.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Benchmark block jobs +# +# Copyright (c) 2019 Virtuozzo International GmbH. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + + +import sys +import os +import socket +import json + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.machine import QEMUMachine +from qemu.qmp import QMPConnectError + + +def bench_block_job(cmd, cmd_args, qemu_args): + """Benchmark block-job + + cmd -- qmp command to run block-job (like blockdev-backup) + cmd_args -- dict of qmp command arguments + qemu_args -- list of Qemu command line arguments, including path to Qemu + binary + + Returns {'seconds': int} on success and {'error': str} on failure, dict may + contain addional 'vm-log' field. Return value is compatible with + simplebench lib. + """ + + vm = QEMUMachine(qemu_args[0], args=qemu_args[1:]) + + try: + vm.launch() + except OSError as e: + return {'error': 'popen failed: ' + str(e)} + except (QMPConnectError, socket.timeout): + return {'error': 'qemu failed: ' + str(vm.get_log())} + + try: + res = vm.qmp(cmd, **cmd_args) + if res != {'return': {}}: + vm.shutdown() + return {'error': '"{}" command failed: {}'.format(cmd, str(res))} + + e = vm.event_wait('JOB_STATUS_CHANGE') + assert e['data']['status'] == 'created' + start_ms = e['timestamp']['seconds'] * 1000000 + \ + e['timestamp']['microseconds'] + + e = vm.events_wait((('BLOCK_JOB_READY', None), + ('BLOCK_JOB_COMPLETED', None), + ('BLOCK_JOB_FAILED', None)), timeout=True) + if e['event'] not in ('BLOCK_JOB_READY', 'BLOCK_JOB_COMPLETED'): + vm.shutdown() + return {'error': 'block-job failed: ' + str(e), + 'vm-log': vm.get_log()} + end_ms = e['timestamp']['seconds'] * 1000000 + \ + e['timestamp']['microseconds'] + finally: + vm.shutdown() + + return {'seconds': (end_ms - start_ms) / 1000000.0} + + +# Bench backup or mirror +def bench_block_copy(qemu_binary, cmd, source, target): + """Helper to run bench_block_job() for mirror or backup""" + assert cmd in ('blockdev-backup', 'blockdev-mirror') + + source['node-name'] = 'source' + target['node-name'] = 'target' + + return bench_block_job(cmd, + {'job-id': 'job0', 'device': 'source', + 'target': 'target', 'sync': 'full'}, + [qemu_binary, + '-blockdev', json.dumps(source), + '-blockdev', json.dumps(target)]) + + +def drv_file(filename): + return {'driver': 'file', 'filename': filename, + 'cache': {'direct': True}, 'aio': 'native'} + + +def drv_nbd(host, port): + return {'driver': 'nbd', + 'server': {'type': 'inet', 'host': host, 'port': port}} + + +if __name__ == '__main__': + import sys + + if len(sys.argv) < 4: + print('USAGE: {} <qmp block-job command name> ' + '<json string of arguments for the command> ' + '<qemu binary path and arguments>'.format(sys.argv[0])) + exit(1) + + res = bench_block_job(sys.argv[1], json.loads(sys.argv[2]), sys.argv[3:]) + if 'seconds' in res: + print('{:.2f}'.format(res['seconds'])) + else: + print(res) diff --git a/scripts/simplebench/simplebench.py b/scripts/simplebench/simplebench.py new file mode 100644 index 0000000000..59e7314ff6 --- /dev/null +++ b/scripts/simplebench/simplebench.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Simple benchmarking framework +# +# Copyright (c) 2019 Virtuozzo International GmbH. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + + +def bench_one(test_func, test_env, test_case, count=5, initial_run=True): + """Benchmark one test-case + + test_func -- benchmarking function with prototype + test_func(env, case), which takes test_env and test_case + arguments and returns {'seconds': int} (which is benchmark + result) on success and {'error': str} on error. Returned + dict may contain any other additional fields. + test_env -- test environment - opaque first argument for test_func + test_case -- test case - opaque second argument for test_func + count -- how many times to call test_func, to calculate average + initial_run -- do initial run of test_func, which don't get into result + + Returns dict with the following fields: + 'runs': list of test_func results + 'average': average seconds per run (exists only if at least one run + succeeded) + 'delta': maximum delta between test_func result and the average + (exists only if at least one run succeeded) + 'n-failed': number of failed runs (exists only if at least one run + failed) + """ + if initial_run: + print(' #initial run:') + print(' ', test_func(test_env, test_case)) + + runs = [] + for i in range(count): + print(' #run {}'.format(i+1)) + res = test_func(test_env, test_case) + print(' ', res) + runs.append(res) + + result = {'runs': runs} + + successed = [r for r in runs if ('seconds' in r)] + if successed: + avg = sum(r['seconds'] for r in successed) / len(successed) + result['average'] = avg + result['delta'] = max(abs(r['seconds'] - avg) for r in successed) + + if len(successed) < count: + result['n-failed'] = count - len(successed) + + return result + + +def ascii_one(result): + """Return ASCII representation of bench_one() returned dict.""" + if 'average' in result: + s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta']) + if 'n-failed' in result: + s += '\n({} failed)'.format(result['n-failed']) + return s + else: + return 'FAILED' + + +def bench(test_func, test_envs, test_cases, *args, **vargs): + """Fill benchmark table + + test_func -- benchmarking function, see bench_one for description + test_envs -- list of test environments, see bench_one + test_cases -- list of test cases, see bench_one + args, vargs -- additional arguments for bench_one + + Returns dict with the following fields: + 'envs': test_envs + 'cases': test_cases + 'tab': filled 2D array, where cell [i][j] is bench_one result for + test_cases[i] for test_envs[j] (i.e., rows are test cases and + columns are test environments) + """ + tab = {} + results = { + 'envs': test_envs, + 'cases': test_cases, + 'tab': tab + } + n = 1 + n_tests = len(test_envs) * len(test_cases) + for env in test_envs: + for case in test_cases: + print('Testing {}/{}: {} :: {}'.format(n, n_tests, + env['id'], case['id'])) + if case['id'] not in tab: + tab[case['id']] = {} + tab[case['id']][env['id']] = bench_one(test_func, env, case, + *args, **vargs) + n += 1 + + print('Done') + return results + + +def ascii(results): + """Return ASCII representation of bench() returned dict.""" + from tabulate import tabulate + + tab = [[""] + [c['id'] for c in results['envs']]] + for case in results['cases']: + row = [case['id']] + for env in results['envs']: + row.append(ascii_one(results['tab'][case['id']][env['id']])) + tab.append(row) + + return tabulate(tab) |