diff options
| author | Markus Armbruster <armbru@redhat.com> | 2019-10-18 09:43:44 +0200 |
|---|---|---|
| committer | Markus Armbruster <armbru@redhat.com> | 2019-10-22 13:53:55 +0200 |
| commit | e6c42b96b9a0fa58cf49bb85cdf473d87fabbeb6 (patch) | |
| tree | c7c740c5013cd4ad7d28f5dbcbf0af14a9e69348 | |
| parent | 61bfb2e1a4666817b9d94f0a96109f8ef51b812b (diff) | |
qapi: Split up scripts/qapi/common.py
The QAPI code generator clocks in at some 3100 SLOC in 8 source files.
Almost 60% of the code is in qapi/common.py. Split it into more
focused modules:
* Move QAPISchemaPragma and QAPISourceInfo to qapi/source.py.
* Move QAPIError and its sub-classes to qapi/error.py.
* Move QAPISchemaParser and QAPIDoc to parser.py. Use the opportunity
to put QAPISchemaParser first.
* Move check_expr() & friends to qapi/expr.py. Use the opportunity to
put the code into a more sensible order.
* Move QAPISchema & friends to qapi/schema.py
* Move QAPIGen and its sub-classes, ifcontext,
QAPISchemaModularCVisitor, and QAPISchemaModularCVisitor to qapi/gen.py
* Delete camel_case(), it's unused since commit e98859a9b9 "qapi:
Clean up after recent conversions to QAPISchemaVisitor"
A number of helper functions remain in qapi/common.py. I considered
moving the code generator helpers to qapi/gen.py, but decided not to.
Perhaps we should rewrite them as methods of QAPIGen some day.
Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20191018074345.24034-7-armbru@redhat.com>
[Add "# -*- coding: utf-8 -*-" lines]
| -rw-r--r-- | Makefile | 13 | ||||
| -rwxr-xr-x | scripts/qapi-gen.py | 10 | ||||
| -rw-r--r-- | scripts/qapi/commands.py | 1 | ||||
| -rw-r--r-- | scripts/qapi/common.py | 2321 | ||||
| -rwxr-xr-x | scripts/qapi/doc.py | 7 | ||||
| -rw-r--r-- | scripts/qapi/error.py | 43 | ||||
| -rw-r--r-- | scripts/qapi/events.py | 2 | ||||
| -rw-r--r-- | scripts/qapi/expr.py | 378 | ||||
| -rw-r--r-- | scripts/qapi/gen.py | 291 | ||||
| -rw-r--r-- | scripts/qapi/introspect.py | 5 | ||||
| -rw-r--r-- | scripts/qapi/parser.py | 570 | ||||
| -rw-r--r-- | scripts/qapi/schema.py | 1043 | ||||
| -rw-r--r-- | scripts/qapi/source.py | 67 | ||||
| -rw-r--r-- | scripts/qapi/types.py | 2 | ||||
| -rw-r--r-- | scripts/qapi/visit.py | 2 | ||||
| -rw-r--r-- | tests/Makefile.include | 13 | ||||
| -rwxr-xr-x | tests/qapi-schema/test-qapi.py | 6 |
17 files changed, 2439 insertions, 2335 deletions
@@ -582,13 +582,20 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated qemu-keymap$(EXESUF): LIBS += $(XKBCOMMON_LIBS) qemu-keymap$(EXESUF): QEMU_CFLAGS += $(XKBCOMMON_CFLAGS) -qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \ +qapi-py = $(SRC_PATH)/scripts/qapi/__init__.py \ +$(SRC_PATH)/scripts/qapi/commands.py \ +$(SRC_PATH)/scripts/qapi/common.py \ +$(SRC_PATH)/scripts/qapi/doc.py \ +$(SRC_PATH)/scripts/qapi/error.py \ $(SRC_PATH)/scripts/qapi/events.py \ +$(SRC_PATH)/scripts/qapi/expr.py \ +$(SRC_PATH)/scripts/qapi/gen.py \ $(SRC_PATH)/scripts/qapi/introspect.py \ +$(SRC_PATH)/scripts/qapi/parser.py \ +$(SRC_PATH)/scripts/qapi/schema.py \ +$(SRC_PATH)/scripts/qapi/source.py \ $(SRC_PATH)/scripts/qapi/types.py \ $(SRC_PATH)/scripts/qapi/visit.py \ -$(SRC_PATH)/scripts/qapi/common.py \ -$(SRC_PATH)/scripts/qapi/doc.py \ $(SRC_PATH)/scripts/qapi-gen.py qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h \ diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py index 3d98ca2e0c..f93f3c7c23 100755 --- a/scripts/qapi-gen.py +++ b/scripts/qapi-gen.py @@ -5,16 +5,18 @@ # See the COPYING file in the top-level directory. from __future__ import print_function + import argparse import re import sys -from qapi.common import QAPIError, QAPISchema -from qapi.types import gen_types -from qapi.visit import gen_visit + from qapi.commands import gen_commands +from qapi.doc import gen_doc from qapi.events import gen_events from qapi.introspect import gen_introspect -from qapi.doc import gen_doc +from qapi.schema import QAPIError, QAPISchema +from qapi.types import gen_types +from qapi.visit import gen_visit def main(argv): diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index 7e3dd1068a..898516b086 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -14,6 +14,7 @@ See the COPYING file in the top-level directory. """ from qapi.common import * +from qapi.gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext def gen_command_decl(name, arg_type, boxed, ret_type): diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py index 306857f0c0..e00dcafce7 100644 --- a/scripts/qapi/common.py +++ b/scripts/qapi/common.py @@ -11,2056 +11,8 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from __future__ import print_function -from contextlib import contextmanager -import copy -import errno -import os import re import string -import sys -from collections import OrderedDict - - -# -# Parsing the schema into expressions -# - - -class QAPISchemaPragma(object): - def __init__(self): - # Are documentation comments required? - self.doc_required = False - # Whitelist of commands allowed to return a non-dictionary - self.returns_whitelist = [] - # Whitelist of entities allowed to violate case conventions - self.name_case_whitelist = [] - - -class QAPISourceInfo(object): - def __init__(self, fname, line, parent): - self.fname = fname - self.line = line - self.parent = parent - self.pragma = parent.pragma if parent else QAPISchemaPragma() - self.defn_meta = None - self.defn_name = None - - def set_defn(self, meta, name): - self.defn_meta = meta - self.defn_name = name - - def next_line(self): - info = copy.copy(self) - info.line += 1 - return info - - def loc(self): - if self.fname is None: - return sys.argv[0] - ret = self.fname - if self.line is not None: - ret += ':%d' % self.line - return ret - - def in_defn(self): - if self.defn_name: - return "%s: In %s '%s':\n" % (self.fname, - self.defn_meta, self.defn_name) - return '' - - def include_path(self): - ret = '' - parent = self.parent - while parent: - ret = 'In file included from %s:\n' % parent.loc() + ret - parent = parent.parent - return ret - - def __str__(self): - return self.include_path() + self.in_defn() + self.loc() - - -class QAPIError(Exception): - def __init__(self, info, col, msg): - Exception.__init__(self) - self.info = info - self.col = col - self.msg = msg - - def __str__(self): - loc = str(self.info) - if self.col is not None: - assert self.info.line is not None - loc += ':%s' % self.col - return loc + ': ' + self.msg - - -class QAPIParseError(QAPIError): - def __init__(self, parser, msg): - col = 1 - for ch in parser.src[parser.line_pos:parser.pos]: - if ch == '\t': - col = (col + 7) % 8 + 1 - else: - col += 1 - QAPIError.__init__(self, parser.info, col, msg) - - -class QAPISemError(QAPIError): - def __init__(self, info, msg): - QAPIError.__init__(self, info, None, msg) - - -class QAPIDoc(object): - """ - A documentation comment block, either definition or free-form - - Definition documentation blocks consist of - - * a body section: one line naming the definition, followed by an - overview (any number of lines) - - * argument sections: a description of each argument (for commands - and events) or member (for structs, unions and alternates) - - * features sections: a description of each feature flag - - * additional (non-argument) sections, possibly tagged - - Free-form documentation blocks consist only of a body section. - """ - - class Section(object): - def __init__(self, name=None): - # optional section name (argument/member or section name) - self.name = name - # the list of lines for this section - self.text = '' - - def append(self, line): - self.text += line.rstrip() + '\n' - - class ArgSection(Section): - def __init__(self, name): - QAPIDoc.Section.__init__(self, name) - self.member = None - - def connect(self, member): - self.member = member - - def __init__(self, parser, info): - # self._parser is used to report errors with QAPIParseError. The - # resulting error position depends on the state of the parser. - # It happens to be the beginning of the comment. More or less - # servicable, but action at a distance. - self._parser = parser - self.info = info - self.symbol = None - self.body = QAPIDoc.Section() - # dict mapping parameter name to ArgSection - self.args = OrderedDict() - self.features = OrderedDict() - # a list of Section - self.sections = [] - # the current section - self._section = self.body - self._append_line = self._append_body_line - - def has_section(self, name): - """Return True if we have a section with this name.""" - for i in self.sections: - if i.name == name: - return True - return False - - def append(self, line): - """ - Parse a comment line and add it to the documentation. - - The way that the line is dealt with depends on which part of - the documentation we're parsing right now: - * The body section: ._append_line is ._append_body_line - * An argument section: ._append_line is ._append_args_line - * A features section: ._append_line is ._append_features_line - * An additional section: ._append_line is ._append_various_line - """ - line = line[1:] - if not line: - self._append_freeform(line) - return - - if line[0] != ' ': - raise QAPIParseError(self._parser, "missing space after #") - line = line[1:] - self._append_line(line) - - def end_comment(self): - self._end_section() - - @staticmethod - def _is_section_tag(name): - return name in ('Returns:', 'Since:', - # those are often singular or plural - 'Note:', 'Notes:', - 'Example:', 'Examples:', - 'TODO:') - - def _append_body_line(self, line): - """ - Process a line of documentation text in the body section. - - If this a symbol line and it is the section's first line, this - is a definition documentation block for that symbol. - - If it's a definition documentation block, another symbol line - begins the argument section for the argument named by it, and - a section tag begins an additional section. Start that - section and append the line to it. - - Else, append the line to the current section. - """ - name = line.split(' ', 1)[0] - # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't - # recognized, and get silently treated as ordinary text - if not self.symbol and not self.body.text and line.startswith('@'): - if not line.endswith(':'): - raise QAPIParseError(self._parser, "line should end with ':'") - self.symbol = line[1:-1] - # FIXME invalid names other than the empty string aren't flagged - if not self.symbol: - raise QAPIParseError(self._parser, "invalid name") - elif self.symbol: - # This is a definition documentation block - if name.startswith('@') and name.endswith(':'): - self._append_line = self._append_args_line - self._append_args_line(line) - elif line == 'Features:': - self._append_line = self._append_features_line - elif self._is_section_tag(name): - self._append_line = self._append_various_line - self._append_various_line(line) - else: - self._append_freeform(line.strip()) - else: - # This is a free-form documentation block - self._append_freeform(line.strip()) - - def _append_args_line(self, line): - """ - Process a line of documentation text in an argument section. - - A symbol line begins the next argument section, a section tag - section or a non-indented line after a blank line begins an - additional section. Start that section and append the line to - it. - - Else, append the line to the current section. - - """ - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): - line = line[len(name)+1:] - self._start_args_section(name[1:-1]) - elif self._is_section_tag(name): - self._append_line = self._append_various_line - self._append_various_line(line) - return - elif (self._section.text.endswith('\n\n') - and line and not line[0].isspace()): - if line == 'Features:': - self._append_line = self._append_features_line - else: - self._start_section() - self._append_line = self._append_various_line - self._append_various_line(line) - return - - self._append_freeform(line.strip()) - - def _append_features_line(self, line): - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): - line = line[len(name)+1:] - self._start_features_section(name[1:-1]) - elif self._is_section_tag(name): - self._append_line = self._append_various_line - self._append_various_line(line) - return - elif (self._section.text.endswith('\n\n') - and line and not line[0].isspace()): - self._start_section() - self._append_line = self._append_various_line - self._append_various_line(line) - return - - self._append_freeform(line.strip()) - - def _append_various_line(self, line): - """ - Process a line of documentation text in an additional section. - - A symbol line is an error. - - A section tag begins an additional section. Start that - section and append the line to it. - - Else, append the line to the current section. - """ - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): - raise QAPIParseError(self._parser, - "'%s' can't follow '%s' section" - % (name, self.sections[0].name)) - elif self._is_section_tag(name): - line = line[len(name)+1:] - self._start_section(name[:-1]) - - if (not self._section.name or - not self._section.name.startswith('Example')): - line = line.strip() - - self._append_freeform(line) - - def _start_symbol_section(self, symbols_dict, name): - # FIXME invalid names other than the empty string aren't flagged - if not name: - raise QAPIParseError(self._parser, "invalid parameter name") - if name in symbols_dict: - raise QAPIParseError(self._parser, - "'%s' parameter name duplicated" % name) - assert not self.sections - self._end_section() - self._section = QAPIDoc.ArgSection(name) - symbols_dict[name] = self._section - - def _start_args_section(self, name): - self._start_symbol_section(self.args, name) - - def _start_features_section(self, name): - self._start_symbol_section(self.features, name) - - def _start_section(self, name=None): - if name in ('Returns', 'Since') and self.has_section(name): - raise QAPIParseError(self._parser, - "duplicated '%s' section" % name) - self._end_section() - self._section = QAPIDoc.Section(name) - self.sections.append(self._section) - - def _end_section(self): - if self._section: - text = self._section.text = self._section.text.strip() - if self._section.name and (not text or text.isspace()): - raise QAPIParseError( - self._parser, - "empty doc section '%s'" % self._section.name) - self._section = None - - def _append_freeform(self, line): - match = re.match(r'(@\S+:)', line) - if match: - raise QAPIParseError(self._parser, - "'%s' not allowed in free-form documentation" - % match.group(1)) - self._section.append(line) - - def connect_member(self, member): - if member.name not in self.args: - # Undocumented TODO outlaw - self.args[member.name] = QAPIDoc.ArgSection(member.name) - self.args[member.name].connect(member) - - def check_expr(self, expr): - if self.has_section('Returns') and 'command' not in expr: - raise QAPISemError(self.info, - "'Returns:' is only valid for commands") - - def check(self): - bogus = [name for name, section in self.args.items() - if not section.member] - if bogus: - raise QAPISemError( - self.info, - "the following documented members are not in " - "the declaration: %s" % ", ".join(bogus)) - - -class QAPISchemaParser(object): - - def __init__(self, fname, previously_included=None, incl_info=None): - previously_included = previously_included or set() - previously_included.add(os.path.abspath(fname)) - - try: - if sys.version_info[0] >= 3: - fp = open(fname, 'r', encoding='utf-8') - else: - fp = open(fname, 'r') - self.src = fp.read() - except IOError as e: - raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), - "can't read %s file '%s': %s" - % ("include" if incl_info else "schema", - fname, - e.strerror)) - - if self.src == '' or self.src[-1] != '\n': - self.src += '\n' - self.cursor = 0 - self.info = QAPISourceInfo(fname, 1, incl_info) - self.line_pos = 0 - self.exprs = [] - self.docs = [] - self.accept() - cur_doc = None - - while self.tok is not None: - info = self.info - if self.tok == '#': - self.reject_expr_doc(cur_doc) - cur_doc = self.get_doc(info) - self.docs.append(cur_doc) - continue - - expr = self.get_expr(False) - if 'include' in expr: - self.reject_expr_doc(cur_doc) - if len(expr) != 1: - raise QAPISemError(info, "invalid 'include' directive") - include = expr['include'] - if not isinstance(include, str): - raise QAPISemError(info, - "value of 'include' must be a string") - incl_fname = os.path.join(os.path.dirname(fname), - include) - self.exprs.append({'expr': {'include': incl_fname}, - 'info': info}) - exprs_include = self._include(include, info, incl_fname, - previously_included) - if exprs_include: - self.exprs.extend(exprs_include.exprs) - self.docs.extend(exprs_include.docs) - elif "pragma" in expr: - self.reject_expr_doc(cur_doc) - if len(expr) != 1: - raise QAPISemError(info, "invalid 'pragma' directive") - pragma = expr['pragma'] - if not isinstance(pragma, dict): - raise QAPISemError( - info, "value of 'pragma' must be an object") - for name, value in pragma.items(): - self._pragma(name, value, info) - else: - expr_elem = {'expr': expr, - 'info': info} - if cur_doc: - if not cur_doc.symbol: - raise QAPISemError( - cur_doc.info, "definition documentation required") - expr_elem['doc'] = cur_doc - self.exprs.append(expr_elem) - cur_doc = None - self.reject_expr_doc(cur_doc) - - @staticmethod - def reject_expr_doc(doc): - if doc and doc.symbol: - raise QAPISemError( - doc.info, - "documentation for '%s' is not followed by the definition" - % doc.symbol) - - def _include(self, include, info, incl_fname, previously_included): - incl_abs_fname = os.path.abspath(incl_fname) - # catch inclusion cycle - inf = info - while inf: - if incl_abs_fname == os.path.abspath(inf.fname): - raise QAPISemError(info, "inclusion loop for %s" % include) - inf = inf.parent - - # skip multiple include of the same file - if incl_abs_fname in previously_included: - return None - - return QAPISchemaParser(incl_fname, previously_included, info) - - def _pragma(self, name, value, info): - if name == 'doc-required': - if not isinstance(value, bool): - raise QAPISemError(info, - "pragma 'doc-required' must be boolean") - info.pragma.doc_required = value - elif name == 'returns-whitelist': - if (not isinstance(value, list) - or any([not isinstance(elt, str) for elt in value])): - raise QAPISemError( - info, - "pragma returns-whitelist must be a list of strings") - info.pragma.returns_whitelist = value - elif name == 'name-case-whitelist': - if (not isinstance(value, list) - or any([not isinstance(elt, str) for elt in value])): - raise QAPISemError( - info, - "pragma name-case-whitelist must be a list of strings") - info.pragma.name_case_whitelist = value - else: - raise QAPISemError(info, "unknown pragma '%s'" % name) - - def accept(self, skip_comment=True): - while True: - self.tok = self.src[self.cursor] - self.pos = self.cursor - self.cursor += 1 - self.val = None - - if self.tok == '#': - if self.src[self.cursor] == '#': - # Start of doc comment - skip_comment = False - self.cursor = self.src.find('\n', self.cursor) - if not skip_comment: - self.val = self.src[self.pos:self.cursor] - return - elif self.tok in '{}:,[]': - return - elif self.tok == "'": - # Note: we accept only printable ASCII - string = '' - esc = False - while True: - ch = self.src[self.cursor] - self.cursor += 1 - if ch == '\n': - raise QAPIParseError(self, "missing terminating \"'\"") - if esc: - # Note: we recognize only \\ because we have - # no use for funny characters in strings - if ch != '\\': - raise QAPIParseError(self, - "unknown escape \\%s" % ch) - esc = False - elif ch == '\\': - esc = True - continue - elif ch == "'": - self.val = string - return - if ord(ch) < 32 or ord(ch) >= 127: - raise QAPIParseError( - self, "funny character in string") - string += ch - elif self.src.startswith('true', self.pos): - self.val = True - self.cursor += 3 - return - elif self.src.startswith('false', self.pos): - self.val = False - self.cursor += 4 - return - elif self.tok == '\n': - if self.cursor == len(self.src): - self.tok = None - return - self.info = self.info.next_line() - self.line_pos = self.cursor - elif not self.tok.isspace(): - # Show up to next structural, whitespace or quote - # character - match = re.match('[^[\\]{}:,\\s\'"]+', - self.src[self.cursor-1:]) - raise QAPIParseError(self, "stray '%s'" % match.group(0)) - - def get_members(self): - expr = OrderedDict() - if self.tok == '}': - self.accept() - return expr - if self.tok != "'": - raise QAPIParseError(self, "expected string or '}'") - while True: - key = self.val - self.accept() - if self.tok != ':': - raise QAPIParseError(self, "expected ':'") - self.accept() - if key in expr: - raise QAPIParseError(self, "duplicate key '%s'" % key) - expr[key] = self.get_expr(True) - if self.tok == '}': - self.accept() - return expr - if self.tok != ',': - raise QAPIParseError(self, "expected ',' or '}'") - self.accept() - if self.tok != "'": - raise QAPIParseError(self, "expected string") - - def get_values(self): - expr = [] - if self.tok == ']': - self.accept() - return expr - if self.tok not in "{['tfn": - raise QAPIParseError( - self, "expected '{', '[', ']', string, boolean or 'null'") - while True: - expr.append(self.get_expr(True)) - if self.tok == ']': - self.accept() - return expr - if self.tok != ',': - raise QAPIParseError(self, "expected ',' or ']'") - self.accept() - - def get_expr(self, nested): - if self.tok != '{' and not nested: - raise QAPIParseError(self, "expected '{'") - if self.tok == '{': - self.accept() - expr = self.get_members() - elif self.tok == '[': - self.accept() - expr = self.get_values() - elif self.tok in "'tfn": - expr = self.val - self.accept() - else: - raise QAPIParseError( - self, "expected '{', '[', string, boolean or 'null'") - return expr - - def get_doc(self, info): - if self.val != '##': - raise QAPIParseError( - self, "junk after '##' at start of documentation comment") - - doc = QAPIDoc(self, info) - self.accept(False) - while self.tok == '#': - if self.val.startswith('##'): - # End of doc comment - if self.val != '##': - raise QAPIParseError( - self, - "junk after '##' at end of documentation comment") - doc.end_comment() - self.accept() - return doc - else: - doc.append(self.val) - self.accept(False) - - raise QAPIParseError(self, "documentation comment must end with '##'") - - -# -# Check (context-free) schema expression structure -# - -# Names must be letters, numbers, -, and _. They must start with letter, -# except for downstream extensions which must start with __RFQDN_. -# Dots are only valid in the downstream extension prefix. -valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?' - '[a-zA-Z][a-zA-Z0-9_-]*$') - - -def check_name_is_str(name, info, source): - if not isinstance(name, str): - raise QAPISemError(info, "%s requires a string name" % source) - - -def check_name_str(name, info, source, - allow_optional=False, enum_member=False, - permit_upper=False): - global valid_name - membername = name - - if allow_optional and name.startswith('*'): - membername = name[1:] - # Enum members can start with a digit, because the generated C - # code always prefixes it with the enum name - if enum_member and membername[0].isdigit(): - membername = 'D' + membername - # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' - # and 'q_obj_*' implicit type names. - if not valid_name.match(membername) or \ - c_name(membername, False).startswith('q_'): - raise QAPISemError(info, "%s has an invalid name" % source) - if not permit_upper and name.lower() != name: - raise QAPISemError( - info, "%s uses uppercase in name" % source) - assert not membername.startswith('*') - - -def check_defn_name_str(name, info, meta): - check_name_str(name, info, meta, permit_upper=True) - if name.endswith('Kind') or name.endswith('List'): - raise QAPISemError( - info, "%s name should not end in '%s'" % (meta, name[-4:])) - - -def check_if(expr, info, source): - - def check_if_str(ifcond, info): - if not isinstance(ifcond, str): - raise QAPISemError( - info, - "'if' condition of %s must be a string or a list of strings" - % source) - if ifcond.strip() == '': - raise QAPISemError( - info, - "'if' condition '%s' of %s makes no sense" - % (ifcond, source)) - - ifcond = expr.get('if') - if ifcond is None: - return - if isinstance(ifcond, list): - if ifcond == []: - raise QAPISemError( - info, "'if' condition [] of %s is useless" % source) - for elt in ifcond: - check_if_str(elt, info) - else: - check_if_str(ifcond, info) - - -def check_type(value, info, source, - allow_array=False, allow_dict=False): - if value is None: - return - - # Array type - if isinstance(value, list): - if not allow_array: - raise QAPISemError(info, "%s cannot be an array" % source) - if len(value) != 1 or not isinstance(value[0], str): - raise QAPISemError(info, - "%s: array type must contain single type name" % - source) - return - - # Type name - if isinstance(value, str): - return - - # Anonymous type - - if not allow_dict: - raise QAPISemError(info, "%s should be a type name" % source) - - if not isinstance(value, OrderedDict): - raise QAPISemError(info, - "%s should be an object or type name" % source) - - permit_upper = allow_dict in info.pragma.name_case_whitelist - - # value is a dictionary, check that each member is okay - for (key, arg) in value.items(): - key_source = "%s member '%s'" % (source, key) - check_name_str(key, info, key_source, - allow_optional=True, permit_upper=permit_upper) - if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): - raise QAPISemError(info, "%s uses reserved name" % key_source) - check_keys(arg, info, key_source, ['type'], ['if']) - check_if(arg, info, key_source) - normalize_if(arg) - check_type(arg['type'], info, key_source, allow_array=True) - - -def check_command(expr, info): - args = expr.get('data') - rets = expr.get('returns') - boxed = expr.get('boxed', False) - - if boxed and args is None: - raise QAPISemError(info, "'boxed': true requires 'data'") - check_type(args, info, "'data'", allow_dict=not boxed) - check_type(rets, info, "'returns'", allow_array=True) - - -def check_event(expr, info): - args = expr.get('data') - boxed = expr.get('boxed', False) - - if boxed and args is None: - raise QAPISemError(info, "'boxed': true requires 'data'") - check_type(args, info, "'data'", allow_dict=not boxed) - - -def check_union(expr, info): - name = expr['union'] - base = expr.get('base') - discriminator = expr.get('discriminator') - members = expr['data'] - - if discriminator is None: # simple union - if base is not None: - raise QAPISemError(info, "'base' requires 'discriminator'") - else: # flat union - check_type(base, info, "'base'", allow_dict=name) - if not base: - raise QAPISemError(info, "'discriminator' requires 'base'") - check_name_is_str(discriminator, info, "'discriminator'") - - for (key, value) in members.items(): - source = "'data' member '%s'" % key - check_name_str(key, info, source) - check_keys(value, info, source, ['type'], ['if']) - check_if(value, info, source) - normalize_if(value) - check_type(value['type'], info, source, allow_array=not base) - - -def check_alternate(expr, info): - members = expr['data'] - - if len(members) == 0: - raise QAPISemError(info, "'data' must not be empty") - for (key, value) in members.items(): - source = "'data' member '%s'" % key - check_name_str(key, info, source) - check_keys(value, info, source, ['type'], ['if']) - check_if(value, info, source) - normalize_if(value) - check_type(value['type'], info, source) - - -def check_enum(expr, info): - name = expr['enum'] - members = expr['data'] - prefix = expr.get('prefix') - - if not isinstance(members, list): - raise QAPISemError(info, "'data' must be an array") - if prefix is not None and not isinstance(prefix, str): - raise QAPISemError(info, "'prefix' must be a string") - - permit_upper = name in info.pragma.name_case_whitelist - - for member in members: - source = "'data' member" - check_keys(member, info, source, ['name'], ['if']) - check_name_is_str(member['name'], info, source) - source = "%s '%s'" % (source, member['name']) - check_name_str(member['name'], info, source, - enum_member=True, permit_upper=permit_upper) - check_if(member, info, source) - normalize_if(member) - - -def check_struct(expr, info): - name = expr['struct'] - members = expr['data'] - features = expr.get('features') - - check_type(members, info, "'data'", allow_dict=name) - check_type(expr.get('base'), info, "'base'") - - if features: - if not isinstance(features, list): - raise QAPISemError(info, "'features' must be an array") - for f in features: - source = "'features' member" - assert isinstance(f, dict) - check_keys(f, info, source, ['name'], ['if']) - check_name_is_str(f['name'], info, source) - source = "%s '%s'" % (source, f['name']) - check_name_str(f['name'], info, source) - check_if(f, info, source) - normalize_if(f) - - -def check_keys(value, info, source, required, optional): - - def pprint(elems): - return ', '.join("'" + e + "'" for e in sorted(elems)) - - missing = set(required) - set(value) - if missing: - raise QAPISemError( - info, - "%s misses key%s %s" - % (source, 's' if len(missing) > 1 else '', - pprint(missing))) - allowed = set(required + optional) - unknown = set(value) - allowed - if unknown: - raise QAPISemError( - info, - "%s has unknown key%s %s\nValid keys are %s." - % (source, 's' if len(unknown) > 1 else '', - pprint(unknown), pprint(allowed))) - - -def check_flags(expr, info): - for key in ['gen', 'success-response']: - if key in expr and expr[key] is not False: - raise QAPISemError( - info, "flag '%s' may only use false value" % key) - for key in ['boxed', 'allow-oob', 'allow-preconfig']: - if key in expr and expr[key] is not True: - raise QAPISemError( - info, "flag '%s' may only use true value" % key) - - -def normalize_enum(expr): - if isinstance(expr['data'], list): - expr['data'] = [m if isinstance(m, dict) else {'name': m} - for m in expr['data']] - - -def normalize_members(members): - if isinstance(members, OrderedDict): - for key, arg in members.items(): - if isinstance(arg, dict): - continue - members[key] = {'type': arg} - - -def normalize_features(features): - if isinstance(features, list): - features[:] = [f if isinstance(f, dict) else {'name': f} - for f in features] - - -def normalize_if(expr): - ifcond = expr.get('if') - if isinstance(ifcond, str): - expr['if'] = [ifcond] - - -def check_exprs(exprs): - for expr_elem in exprs: - expr = expr_elem['expr'] - info = expr_elem['info'] - doc = expr_elem.get('doc') - - if 'include' in expr: - continue - - if 'enum' in expr: - meta = 'enum' - elif 'union' in expr: - meta = 'union' - elif 'alternate' in expr: - meta = 'alternate' - elif 'struct' in expr: - meta = 'struct' - elif 'command' in expr: - meta = 'command' - elif 'event' in expr: - meta = 'event' - else: - raise QAPISemError(info, "expression is missing metatype") - - name = expr[meta] - check_name_is_str(name, info, "'%s'" % meta) - info.set_defn(meta, name) - check_defn_name_str(name, info, meta) - - if doc: - if doc.symbol != name: - raise QAPISemError( - info, "documentation comment is for '%s'" % doc.symbol) - doc.check_expr(expr) - elif info.pragma.doc_required: - raise QAPISemError(info, - "documentation comment required") - - if meta == 'enum': - check_keys(expr, info, meta, - ['enum', 'data'], ['if', 'prefix']) - normalize_enum(expr) - check_enum(expr, info) - elif meta == 'union': - check_keys(expr, info, meta, - ['union', 'data'], - ['base', 'discriminator', 'if']) - normalize_members(expr.get('base')) - normalize_members(expr['data']) - check_union(expr, info) - elif meta == 'alternate': - check_keys(expr, info, meta, - ['alternate', 'data'], ['if']) - normalize_members(expr['data']) - check_alternate(expr, info) - elif meta == 'struct': - check_keys(expr, info, meta, - ['struct', 'data'], ['base', 'if', 'features']) - normalize_members(expr['data']) - normalize_features(expr.get('features')) - check_struct(expr, info) - elif meta == 'command': - check_keys(expr, info, meta, - ['command'], - ['data', 'returns', 'boxed', 'if', - 'gen', 'success-response', 'allow-oob', - 'allow-preconfig']) - normalize_members(expr.get('data')) - check_command(expr, info) - elif meta == 'event': - check_keys(expr, info, meta, - ['event'], ['data', 'boxed', 'if']) - normalize_members(expr.get('data')) - check_event(expr, info) - else: - assert False, 'unexpected meta type' - - normalize_if(expr) - check_if(expr, info, meta) - check_flags(expr, info) - - return exprs - - -# -# Schema compiler frontend -# TODO catching name collisions in generated code would be nice -# - -class QAPISchemaEntity(object): - meta = None - - def __init__(self, name, info, doc, ifcond=None): - assert name is None or isinstance(name, str) - self.name = name - self._module = None - # For explicitly defined entities, info points to the (explicit) - # definition. For builtins (and their arrays), info is None. - # For implicitly defined entities, info points to a place that - # triggered the implicit definition (there may be more than one - # such place). - self.info = info - self.doc = doc - self._ifcond = ifcond or [] - self._checked = False - - def c_name(self): - return c_name(self.name) - - def check(self, schema): - assert not self._checked - if self.info: - self._module = os.path.relpath(self.info.fname, - os.path.dirname(schema.fname)) - self._checked = True - - @property - def ifcond(self): - assert self._checked - return self._ifcond - - @property - def module(self): - assert self._checked - return self._module - - def is_implicit(self): - return not self.info - - def visit(self, visitor): - assert self._checked - - def describe(self): - assert self.meta - return "%s '%s'" % (self.meta, self.name) - - -class QAPISchemaVisitor(object): - def visit_begin(self, schema): - pass - - def visit_end(self): - pass - - def visit_module(self, fname): - pass - - def visit_needed(self, entity): - # Default to visiting everything - return True - - def visit_include(self, fname, info): - pass - - def visit_builtin_type(self, name, info, json_type): - pass - - def visit_enum_type(self, name, info, ifcond, members, prefix): - pass - - def visit_array_type(self, name, info, ifcond, element_type): - pass - - def visit_object_type(self, name, info, ifcond, base, members, variants, - features): - pass - - def visit_object_type_flat(self, name, info, ifcond, members, variants, - features): - pass - - def visit_alternate_type(self, name, info, ifcond, variants): - pass - - def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, - success_response, boxed, allow_oob, allow_preconfig): - pass - - def visit_event(self, name, info, ifcond, arg_type, boxed): - pass - - -class QAPISchemaInclude(QAPISchemaEntity): - - def __init__(self, fname, info): - QAPISchemaEntity.__init__(self, None, info, None) - self.fname = fname - - def visit(self, visitor): - QAPISchemaEntity.visit(self, visitor) - visitor.visit_include(self.fname, self.info) - - -class QAPISchemaType(QAPISchemaEntity): - # Return the C type for common use. - # For the types we commonly box, this is a pointer type. - def c_type(self): - pass - - # Return the C type to be used in a parameter list. - def c_param_type(self): - return self.c_type() - - # Return the C type to be used where we suppress boxing. - def c_unboxed_type(self): - return self.c_type() - - def json_type(self): - pass - - def alternate_qtype(self): - json2qtype = { - 'null': 'QTYPE_QNULL', - 'string': 'QTYPE_QSTRING', - 'number': 'QTYPE_QNUM', - 'int': 'QTYPE_QNUM', - 'boolean': 'QTYPE_QBOOL', - 'object': 'QTYPE_QDICT' - } - return json2qtype.get(self.json_type()) - - def doc_type(self): - if self.is_implicit(): - return None - return self.name - - def describe(self): - assert self.meta - return "%s type '%s'" % (self.meta, self.name) - - -class QAPISchemaBuiltinType(QAPISchemaType): - meta = 'built-in' - - def __init__(self, name, json_type, c_type): - QAPISchemaType.__init__(self, name, None, None) - assert not c_type or isinstance(c_type, str) - assert json_type in ('string', 'number', 'int', 'boolean', 'null', - 'value') - self._json_type_name = json_type - self._c_type_name = c_type - - def c_name(self): - return self.name - - def c_type(self): - return self._c_type_name - - def c_param_type(self): - if self.name == 'str': - return 'const ' + self._c_type_name - return self._c_type_name - - def json_type(self): - return self._json_type_name - - def doc_type(self): - return self.json_type() - - def visit(self, visitor): - QAPISchemaType.visit(self, visitor) - visitor.visit_builtin_type(self.name, self.info, self.json_type()) - - -class QAPISchemaEnumType(QAPISchemaType): - meta = 'enum' - - def __init__(self, name, info, doc, ifcond, members, prefix): - QAPISchemaType.__init__(self, name, info, doc, ifcond) - for m in members: - assert isinstance(m, QAPISchemaEnumMember) - m.set_defined_in(name) - assert prefix is None or isinstance(prefix, str) - self.members = members - self.prefix = prefix - - def check(self, schema): - QAPISchemaType.check(self, schema) - seen = {} - for m in self.members: - m.check_clash(self.info, seen) - if self.doc: - self.doc.connect_member(m) - - def is_implicit(self): - # See QAPISchema._make_implicit_enum_type() and ._def_predefineds() - return self.name.endswith('Kind') or self.name == 'QType' - - def c_type(self): - return c_name(self.name) - - def member_names(self): - return [m.name for m in self.members] - - def json_type(self): - return 'string' - - def visit(self, visitor): - QAPISchemaType.visit(self, visitor) - visitor.visit_enum_type(self.name, self.info, self.ifcond, - self.members, self.prefix) - - -class QAPISchemaArrayType(QAPISchemaType): - meta = 'array' - - def __init__(self, name, info, element_type): - QAPISchemaType.__init__(self, name, info, None, None) - assert isinstance(element_type, str) - self._element_type_name = element_type - self.element_type = None - - def check(self, schema): - QAPISchemaType.check(self, schema) - self.element_type = schema.resolve_type( - self._element_type_name, self.info, - self.info and self.info.defn_meta) - assert not isinstance(self.element_type, QAPISchemaArrayType) - - @property - def ifcond(self): - assert self._checked - return self.element_type.ifcond - - @property - def module(self): - assert self._checked - return self.element_type.module - - def is_implicit(self): - return True - - def c_type(self): - return c_name(self.name) + pointer_suffix - - def json_type(self): - return 'array' - - def doc_type(self): - elt_doc_type = self.element_type.doc_type() - if not elt_doc_type: - return None - return 'array of ' + elt_doc_type - - def visit(self, visitor): - QAPISchemaType.visit(self, visitor) - visitor.visit_array_type(self.name, self.info, self.ifcond, - self.element_type) - - def describe(self): - assert self.meta - return "%s type ['%s']" % (self.meta, self._element_type_name) - - -class QAPISchemaObjectType(QAPISchemaType): - def __init__(self, name, info, doc, ifcond, - base, local_members, variants, features): - # struct has local_members, optional base, and no variants - # flat union has base, variants, and no local_members - # simple union has local_members, variants, and no base - QAPISchemaType.__init__(self, name, info, doc, ifcond) - self.meta = 'union' if variants else 'struct' - assert base is None or isinstance(base, str) - for m in local_members: - assert isinstance(m, QAPISchemaObjectTypeMember) - m.set_defined_in(name) - if variants is not None: - assert isinstance(variants, QAPISchemaObjectTypeVariants) - variants.set_defined_in(name) - for f in features: - assert isinstance(f, QAPISchemaFeature) - f.set_defined_in(name) - self._base_name = base - self.base = None - self.local_members = local_members - self.variants = variants - self.members = None - self.features = features - - def check(self, schema): - # This calls another type T's .check() exactly when the C - # struct emitted by gen_object() contains that T's C struct - # (pointers don't count). - if self.members is not None: - # A previous .check() completed: nothing to do - return - if self._checked: - # Recursed: C struct contains itself - raise QAPISemError(self.info, - "object %s contains itself" % self.name) - - QAPISchemaType.check(self, schema) - assert self._checked and self.members is None - - seen = OrderedDict() - if self._base_name: - self.base = schema.resolve_type(self._base_name, self.info, - "'base'") - if (not isinstance(self.base, QAPISchemaObjectType) - or self.base.variants): - raise QAPISemError( - self.info, - "'base' requires a struct type, %s isn't" - % self.base.describe()) - self.base.check(schema) - self.base.check_clash(self.info, seen) - for m in self.local_members: - m.check(schema) - m.check_clash(self.info, seen) - if self.doc: - self.doc.connect_member(m) - members = seen.values() - - if self.variants: - self.variants.check(schema, seen) - self.variants.check_clash(self.info, seen) - - # Features are in a name space separate from members - seen = {} - for f in self.features: - f.check_clash(self.info, seen) - - if self.doc: - self.doc.check() - - self.members = members # mark completed - - # Check that the members of this type do not cause duplicate JSON members, - # and update seen to track the members seen so far. Report any errors - # on behalf of info, which is not necessarily self.info - def check_clash(self, info, seen): - assert self._checked - assert not self.variants # not implemented - for m in self.members: - m.check_clash(info, seen) - - @property - def ifcond(self): - assert self._checked - if isinstance(self._ifcond, QAPISchemaType): - # Simple union wrapper type inherits from wrapped type; - # see _make_implicit_object_type() - return self._ifcond.ifcond - return self._ifcond - - def is_implicit(self): - # See QAPISchema._make_implicit_object_type(), as well as - # _def_predefineds() - return self.name.startswith('q_') - - def is_empty(self): - assert self.members is not None - return not self.members and not self.variants - - def c_name(self): - assert self.name != 'q_empty' - return QAPISchemaType.c_name(self) - - def c_type(self): - assert not self.is_implicit() - return c_name(self.name) + pointer_suffix - - def c_unboxed_type(self): - return c_name(self.name) - - def json_type(self): - return 'object' - - def visit(self, visitor): - QAPISchemaType.visit(self, visitor) - visitor.visit_object_type(self.name, self.info, self.ifcond, - self.base, self.local_members, self.variants, - self.features) - visitor.visit_object_type_flat(self.name, self.info, self.ifcond, - self.members, self.variants, - self.features) - - -class QAPISchemaMember(object): - """ Represents object members, enum members and features """ - role = 'member' - - def __init__(self, name, info, ifcond=None): - assert isinstance(name, str) - self.name = name - self.info = info - self.ifcond = ifcond or [] - self.defined_in = None - - def set_defined_in(self, name): - assert not self.defined_in - self.defined_in = name - - def check_clash(self, info, seen): - cname = c_name(self.name) - if cname in seen: - raise QAPISemError( - info, - "%s collides with %s" - % (self.describe(info), seen[cname].describe(info))) - seen[cname] = self - - def describe(self, info): - role = self.role - defined_in = self.defined_in - assert defined_in - - if defined_in.startswith('q_obj_'): - # See QAPISchema._make_implicit_object_type() - reverse the - # mapping there to create a nice human-readable description - defined_in = defined_in[6:] - if defined_in.endswith('-arg'): - # Implicit type created for a command's dict 'data' - assert role == 'member' - role = 'parameter' - elif defined_in.endswith('-base'): - # Implicit type created for a flat union's dict 'base' - role = 'base ' + role - else: - # Implicit type created for a simple union's branch - assert defined_in.endswith('-wrapper') - # Unreachable and not implemented - assert False - elif defined_in.endswith('Kind'): - # See QAPISchema._make_implicit_enum_type() - # Implicit enum created for simple union's branches - assert role == 'value' - role = 'branch' - elif defined_in != info.defn_name: - return "%s '%s' of type '%s'" % (role, self.name, defined_in) - return "%s '%s'" % (role, self.name) - - -class QAPISchemaEnumMember(QAPISchemaMember): - role = 'value' - - -class QAPISchemaFeature(QAPISchemaMember): - role = 'feature' - - -class QAPISchemaObjectTypeMember(QAPISchemaMember): - def __init__(self, name, info, typ, optional, ifcond=None): - QAPISchemaMember.__init__(self, name, info, ifcond) - assert isinstance(typ, str) - assert isinstance(optional, bool) - self._type_name = typ - self.type = None - self.optional = optional - - def check(self, schema): - assert self.defined_in - self.type = schema.resolve_type(self._type_name, self.info, - self.describe) - - -class QAPISchemaObjectTypeVariants(object): - def __init__(self, tag_name, info, tag_member, variants): - # Flat unions pass tag_name but not tag_member. - # Simple unions and alternates pass tag_member but not tag_name. - # After check(), tag_member is always set, and tag_name remains - # a reliable witness of being used by a flat union. - assert bool(tag_member) != bool(tag_name) - assert (isinstance(tag_name, str) or - isinstance(tag_member, QAPISchemaObjectTypeMember)) - for v in variants: - assert isinstance(v, QAPISchemaObjectTypeVariant) - self._tag_name = tag_name - self.info = info - self.tag_member = tag_member - self.variants = variants - - def set_defined_in(self, name): - for v in self.variants: - v.set_defined_in(name) - - def check(self, schema, seen): - if not self.tag_member: # flat union - self.tag_member = seen.get(c_name(self._tag_name)) - base = "'base'" - # Pointing to the base type when not implicit would be - # nice, but we don't know it here - if not self.tag_member or self._tag_name != self.tag_member.name: - raise QAPISemError( - self.info, - "discriminator '%s' is not a member of %s" - % (self._tag_name, base)) - # Here we do: - base_type = schema.lookup_type(self.tag_member.defined_in) - assert base_type - if not base_type.is_implicit(): - base = "base type '%s'" % self.tag_member.defined_in - if not isinstance(self.tag_member.type, QAPISchemaEnumType): - raise QAPISemError( - self.info, - "discriminator member '%s' of %s must be of enum type" - % (self._tag_name, base)) - if self.tag_member.optional: - raise QAPISemError( - self.info, - "discriminator member '%s' of %s must not be optional" - % (self._tag_name, base)) - if self.tag_member.ifcond: - raise QAPISemError( - self.info, - "discriminator member '%s' of %s must not be conditional" - % (self._tag_name, base)) - else: # simple union - assert isinstance(self.tag_member.type, QAPISchemaEnumType) - assert not self.tag_member.optional - assert self.tag_member.ifcond == [] - if self._tag_name: # flat union - # branches that are not explicitly covered get an empty type - cases = set([v.name for v in self.variants]) - for m in self.tag_member.type.members: - if m.name not in cases: - v = QAPISchemaObjectTypeVariant(m.name, self.info, - 'q_empty', m.ifcond) - v.set_defined_in(self.tag_member.defined_in) - self.variants.append(v) - if not self.variants: - raise QAPISemError(self.info, "union has no branches") - for v in self.variants: - v.check(schema) - # Union names must match enum values; alternate names are - # checked separately. Use 'seen' to tell the two apart. - if seen: - if v.name not in self.tag_member.type.member_names(): - raise QAPISemError( - self.info, - "branch '%s' is not a value of %s" - % (v.name, self.tag_member.type.describe())) - if (not isinstance(v.type, QAPISchemaObjectType) - or v.type.variants): - raise QAPISemError( - self.info, - "%s cannot use %s" - % (v.describe(self.info), v.type.describe())) - v.type.check(schema) - - def check_clash(self, info, seen): - for v in self.variants: - # Reset seen map for each variant, since qapi names from one - # branch do not affect another branch - v.type.check_clash(info, dict(seen)) - - -class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember): - role = 'branch' - - def __init__(self, name, info, typ, ifcond=None): - QAPISchemaObjectTypeMember.__init__(self, name, info, typ, - False, ifcond) - - -class QAPISchemaAlternateType(QAPISchemaType): - meta = 'alternate' - - def __init__(self, name, info, doc, ifcond, variants): - QAPISchemaType.__init__(self, name, info, doc, ifcond) - assert isinstance(variants, QAPISchemaObjectTypeVariants) - assert variants.tag_member - variants.set_defined_in(name) - variants.tag_member.set_defined_in(self.name) - self.variants = variants - - def check(self, schema): - QAPISchemaType.check(self, schema) - self.variants.tag_member.check(schema) - # Not calling self.variants.check_clash(), because there's nothing - # to clash with - self.variants.check(schema, {}) - # Alternate branch names have no relation to the tag enum values; - # so we have to check for potential name collisions ourselves. - seen = {} - types_seen = {} - for v in self.variants.variants: - v.check_clash(self.info, seen) - qtype = v.type.alternate_qtype() - if not qtype: - raise QAPISemError( - self.info, - "%s cannot use %s" - % (v.describe(self.info), v.type.describe())) - conflicting = set([qtype]) - if qtype == 'QTYPE_QSTRING': - if isinstance(v.type, QAPISchemaEnumType): - for m in v.type.members: - if m.name in ['on', 'off']: - conflicting.add('QTYPE_QBOOL') - if re.match(r'[-+0-9.]', m.name): - # lazy, could be tightened - conflicting.add('QTYPE_QNUM') - else: - conflicting.add('QTYPE_QNUM') - conflicting.add('QTYPE_QBOOL') - for qt in conflicting: - if qt in types_seen: - raise QAPISemError( - self.info, - "%s can't be distinguished from '%s'" - % (v.describe(self.info), types_seen[qt])) - types_seen[qt] = v.name - if self.doc: - self.doc.connect_member(v) - if self.doc: - self.doc.check() - - def c_type(self): - return c_name(self.name) + pointer_suffix - - def json_type(self): - return 'value' - - def visit(self, visitor): - QAPISchemaType.visit(self, visitor) - visitor.visit_alternate_type(self.name, self.info, self.ifcond, - self.variants) - - -class QAPISchemaCommand(QAPISchemaEntity): - meta = 'command' - - def __init__(self, name, info, doc, ifcond, arg_type, ret_type, - gen, success_response, boxed, allow_oob, allow_preconfig): - QAPISchemaEntity.__init__(self, name, info, doc, ifcond) - assert not arg_type or isinstance(arg_type, str) - assert not ret_type or isinstance(ret_type, str) - self._arg_type_name = arg_type - self.arg_type = None - self._ret_type_name = ret_type - self.ret_type = None - self.gen = gen - self.success_response = success_response - self.boxed = boxed - self.allow_oob = allow_oob - self.allow_preconfig = allow_preconfig - - def check(self, schema): - QAPISchemaEntity.check(self, schema) - if self._arg_type_name: - self.arg_type = schema.resolve_type( - self._arg_type_name, self.info, "command's 'data'") - if not isinstance(self.arg_type, QAPISchemaObjectType): - raise QAPISemError( - self.info, - "command's 'data' cannot take %s" - % self.arg_type.describe()) - if self.arg_type.variants and not self.boxed: - raise QAPISemError( - self.info, - "command's 'data' can take %s only with 'boxed': true" - % self.arg_type.describe()) - if self._ret_type_name: - self.ret_type = schema.resolve_type( - self._ret_type_name, self.info, "command's 'returns'") - if self.name not in self.info.pragma.returns_whitelist: - if not (isinstance(self.ret_type, QAPISchemaObjectType) - or (isinstance(self.ret_type, QAPISchemaArrayType) - and isinstance(self.ret_type.element_type, - QAPISchemaObjectType))): - raise QAPISemError( - self.info, - "command's 'returns' cannot take %s" - % self.ret_type.describe()) - - def visit(self, visitor): - QAPISchemaEntity.visit(self, visitor) - visitor.visit_command(self.name, self.info, self.ifcond, - self.arg_type, self.ret_type, - self.gen, self.success_response, - self.boxed, self.allow_oob, - self.allow_preconfig) - - -class QAPISchemaEvent(QAPISchemaEntity): - meta = 'event' - - def __init__(self, name, info, doc, ifcond, arg_type, boxed): - QAPISchemaEntity.__init__(self, name, info, doc, ifcond) - assert not arg_type or isinstance(arg_type, str) - self._arg_type_name = arg_type - self.arg_type = None - self.boxed = boxed - - def check(self, schema): - QAPISchemaEntity.check(self, schema) - if self._arg_type_name: - self.arg_type = schema.resolve_type( - self._arg_type_name, self.info, "event's 'data'") - if not isinstance(self.arg_type, QAPISchemaObjectType): - raise QAPISemError( - self.info, - "event's 'data' cannot take %s" - % self.arg_type.describe()) - if self.arg_type.variants and not self.boxed: - raise QAPISemError( - self.info, - "event's 'data' can take %s only with 'boxed': true" - % self.arg_type.describe()) - - def visit(self, visitor): - QAPISchemaEntity.visit(self, visitor) - visitor.visit_event(self.name, self.info, self.ifcond, - self.arg_type, self.boxed) - - -class QAPISchema(object): - def __init__(self, fname): - self.fname = fname - parser = QAPISchemaParser(fname) - exprs = check_exprs(parser.exprs) - self.docs = parser.docs - self._entity_list = [] - self._entity_dict = {} - self._predefining = True - self._def_predefineds() - self._predefining = False - self._def_exprs(exprs) - self.check() - - def _def_entity(self, ent): - # Only the predefined types are allowed to not have info - assert ent.info or self._predefining - self._entity_list.append(ent) - if ent.name is None: - return - # TODO reject names that differ only in '_' vs. '.' vs. '-', - # because they're liable to clash in generated C. - other_ent = self._entity_dict.get(ent.name) - if other_ent: - if other_ent.info: - where = QAPIError(other_ent.info, None, "previous definition") - raise QAPISemError( - ent.info, - "'%s' is already defined\n%s" % (ent.name, where)) - raise QAPISemError( - ent.info, "%s is already defined" % other_ent.describe()) - self._entity_dict[ent.name] = ent - - def lookup_entity(self, name, typ=None): - ent = self._entity_dict.get(name) - if typ and not isinstance(ent, typ): - return None - return ent - - def lookup_type(self, name): - return self.lookup_entity(name, QAPISchemaType) - - def resolve_type(self, name, info, what): - typ = self.lookup_type(name) - if not typ: - if callable(what): - what = what(info) - raise QAPISemError( - info, "%s uses unknown type '%s'" % (what, name)) - return typ - - def _def_include(self, expr, info, doc): - include = expr['include'] - assert doc is None - main_info = info - while main_info.parent: - main_info = main_info.parent - fname = os.path.relpath(include, os.path.dirname(main_info.fname)) - self._def_entity(QAPISchemaInclude(fname, info)) - - def _def_builtin_type(self, name, json_type, c_type): - self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type)) - # Instantiating only the arrays that are actually used would - # be nice, but we can't as long as their generated code - # (qapi-builtin-types.[ch]) may be shared by some other - # schema. - self._make_array_type(name, None) - - def _def_predefineds(self): - for t in [('str', 'string', 'char' + pointer_suffix), - ('number', 'number', 'double'), - ('int', 'int', 'int64_t'), - ('int8', 'int', 'int8_t'), - ('int16', 'int', 'int16_t'), - ('int32', 'int', 'int32_t'), - ('int64', 'int', 'int64_t'), - ('uint8', 'int', 'uint8_t'), - ('uint16', 'int', 'uint16_t'), - ('uint32', 'int', 'uint32_t'), - ('uint64', 'int', 'uint64_t'), - ('size', 'int', 'uint64_t'), - ('bool', 'boolean', 'bool'), - ('any', 'value', 'QObject' + pointer_suffix), - ('null', 'null', 'QNull' + pointer_suffix)]: - self._def_builtin_type(*t) - self.the_empty_object_type = QAPISchemaObjectType( - 'q_empty', None, None, None, None, [], None, []) - self._def_entity(self.the_empty_object_type) - - qtypes = ['none', 'qnull', 'qnum', 'qstring', 'qdict', 'qlist', - 'qbool'] - qtype_values = self._make_enum_members( - [{'name': n} for n in qtypes], None) - - self._def_entity(QAPISchemaEnumType('QType', None, None, None, - qtype_values, 'QTYPE')) - - def _make_features(self, features, info): - return [QAPISchemaFeature(f['name'], info, f.get('if')) - for f in features] - - def _make_enum_members(self, values, info): - return [QAPISchemaEnumMember(v['name'], info, v.get('if')) - for v in values] - - def _make_implicit_enum_type(self, name, info, ifcond, values): - # See also QAPISchemaObjectTypeMember.describe() - name = name + 'Kind' # reserved by check_defn_name_str() - self._def_entity(QAPISchemaEnumType( - name, info, None, ifcond, self._make_enum_members(values, info), - None)) - return name - - def _make_array_type(self, element_type, info): - name = element_type + 'List' # reserved by check_defn_name_str() - if not self.lookup_type(name): - self._def_entity(QAPISchemaArrayType(name, info, element_type)) - return name - - def _make_implicit_object_type(self, name, info, doc, ifcond, - role, members): - if not members: - return None - # See also QAPISchemaObjectTypeMember.describe() - name = 'q_obj_%s-%s' % (name, role) - typ = self.lookup_entity(name, QAPISchemaObjectType) - if typ: - # The implicit object type has multiple users. This can - # happen only for simple unions' implicit wrapper types. - # Its ifcond should be the disjunction of its user's - # ifconds. Not implemented. Instead, we always pass the - # wrapped type's ifcond, which is trivially the same for all - # users. It's also necessary for the wrapper to compile. - # But it's not tight: the disjunction need not imply it. We - # may end up compiling useless wrapper types. - # TODO kill simple unions or implement the disjunction - assert (ifcond or []) == typ._ifcond # pylint: disable=protected-access - else: - self._def_entity(QAPISchemaObjectType(name, info, doc, ifcond, - None, members, None, [])) - return name - - def _def_enum_type(self, expr, info, doc): - name = expr['enum'] - data = expr['data'] - prefix = expr.get('prefix') - ifcond = expr.get('if') - self._def_entity(QAPISchemaEnumType( - name, info, doc, ifcond, - self._make_enum_members(data, info), prefix)) - - def _make_member(self, name, typ, ifcond, info): - optional = False - if name.startswith('*'): - name = name[1:] - optional = True - if isinstance(typ, list): - assert len(typ) == 1 - typ = self._make_array_type(typ[0], info) - return QAPISchemaObjectTypeMember(name, info, typ, optional, ifcond) - - def _make_members(self, data, info): - return [self._make_member(key, value['type'], value.get('if'), info) - for (key, value) in data.items()] - - def _def_struct_type(self, expr, info, doc): - name = expr['struct'] - base = expr.get('base') - data = expr['data'] - ifcond = expr.get('if') - features = expr.get('features', []) - self._def_entity(QAPISchemaObjectType( - name, info, doc, ifcond, base, - self._make_members(data, info), - None, - self._make_features(features, info))) - - def _make_variant(self, case, typ, ifcond, info): - return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) - - def _make_simple_variant(self, case, typ, ifcond, info): - if isinstance(typ, list): - assert len(typ) == 1 - typ = self._make_array_type(typ[0], info) - typ = self._make_implicit_object_type( - typ, info, None, self.lookup_type(typ), - 'wrapper', [self._make_member('data', typ, None, info)]) - return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) - - def _def_union_type(self, expr, info, doc): - name = expr['union'] - data = expr['data'] - base = expr.get('base') - ifcond = expr.get('if') - tag_name = expr.get('discriminator') - tag_member = None - if isinstance(base, dict): - base = self._make_implicit_object_type( - name, info, doc, ifcond, - 'base', self._make_members(base, info)) - if tag_name: - variants = [self._make_variant(key, value['type'], - value.get('if'), info) - for (key, value) in data.items()] - members = [] - else: - variants = [self._make_simple_variant(key, value['type'], - value.get('if'), info) - for (key, value) in data.items()] - enum = [{'name': v.name, 'if': v.ifcond} for v in variants] - typ = self._make_implicit_enum_type(name, info, ifcond, enum) - tag_member = QAPISchemaObjectTypeMember('type', info, typ, False) - members = [tag_member] - self._def_entity( - QAPISchemaObjectType(name, info, doc, ifcond, base, members, - QAPISchemaObjectTypeVariants( - tag_name, info, tag_member, variants), - [])) - - def _def_alternate_type(self, expr, info, doc): - name = expr['alternate'] - data = expr['data'] - ifcond = expr.get('if') - variants = [self._make_variant(key, value['type'], value.get('if'), - info) - for (key, value) in data.items()] - tag_member = QAPISchemaObjectTypeMember('type', info, 'QType', False) - self._def_entity( - QAPISchemaAlternateType(name, info, doc, ifcond, - QAPISchemaObjectTypeVariants( - None, info, tag_member, variants))) - - def _def_command(self, expr, info, doc): - name = expr['command'] - data = expr.get('data') - rets = expr.get('returns') - gen = expr.get('gen', True) - success_response = expr.get('success-response', True) - boxed = expr.get('boxed', False) - allow_oob = expr.get('allow-oob', False) - allow_preconfig = expr.get('allow-preconfig', False) - ifcond = expr.get('if') - if isinstance(data, OrderedDict): - data = self._make_implicit_object_type( - name, info, doc, ifcond, 'arg', self._make_members(data, info)) - if isinstance(rets, list): - assert len(rets) == 1 - rets = self._make_array_type(rets[0], info) - self._def_entity(QAPISchemaCommand(name, info, doc, ifcond, data, rets, - gen, success_response, - boxed, allow_oob, allow_preconfig)) - - def _def_event(self, expr, info, doc): - name = expr['event'] - data = expr.get('data') - boxed = expr.get('boxed', False) - ifcond = expr.get('if') - if isinstance(data, OrderedDict): - data = self._make_implicit_object_type( - name, info, doc, ifcond, 'arg', self._make_members(data, info)) - self._def_entity(QAPISchemaEvent(name, info, doc, ifcond, data, boxed)) - - def _def_exprs(self, exprs): - for expr_elem in exprs: - expr = expr_elem['expr'] - info = expr_elem['info'] - doc = expr_elem.get('doc') - if 'enum' in expr: - self._def_enum_type(expr, info, doc) - elif 'struct' in expr: - self._def_struct_type(expr, info, doc) - elif 'union' in expr: - self._def_union_type(expr, info, doc) - elif 'alternate' in expr: - self._def_alternate_type(expr, info, doc) - elif 'command' in expr: - self._def_command(expr, info, doc) - elif 'event' in expr: - self._def_event(expr, info, doc) - elif 'include' in expr: - self._def_include(expr, info, doc) - else: - assert False - - def check(self): - for ent in self._entity_list: - ent.check(self) - - def visit(self, visitor): - visitor.visit_begin(self) - module = None - visitor.visit_module(module) - for entity in self._entity_list: - if visitor.visit_needed(entity): - if entity.module != module: - module = entity.module - visitor.visit_module(module) - entity.visit(visitor) - visitor.visit_end() - - -# -# Code generation helpers -# - -def camel_case(name): - new_name = '' - first = True - for ch in name: - if ch in ['_', '-']: - first = True - elif first: - new_name += ch.upper() - first = False - else: - new_name += ch.lower() - return new_name # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1 @@ -2223,22 +175,6 @@ def gen_endif(ifcond): return ret -def _wrap_ifcond(ifcond, before, after): - if before == after: - return after # suppress empty #if ... #endif - - assert after.startswith(before) - out = before - added = after[len(before):] - if added[0] == '\n': - out += '\n' - added = added[1:] - out += gen_if(ifcond) - out += added - out += gen_endif(ifcond) - return out - - def build_params(arg_type, boxed, extra=None): ret = '' sep = '' @@ -2258,260 +194,3 @@ def build_params(arg_type, boxed, extra=None): if extra: ret += sep + extra return ret if ret else 'void' - - -# -# Accumulate and write output -# - -class QAPIGen(object): - - def __init__(self, fname): - self.fname = fname - self._preamble = '' - self._body = '' - - def preamble_add(self, text): - self._preamble += text - - def add(self, text): - self._body += text - - def get_content(self): - return self._top() + self._preamble + self._body + self._bottom() - - def _top(self): - return '' - - def _bottom(self): - return '' - - def write(self, output_dir): - pathname = os.path.join(output_dir, self.fname) - dir = os.path.dirname(pathname) - if dir: - try: - os.makedirs(dir) - except os.error as e: - if e.errno != errno.EEXIST: - raise - fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666) - if sys.version_info[0] >= 3: - f = open(fd, 'r+', encoding='utf-8') - else: - f = os.fdopen(fd, 'r+') - text = self.get_content() - oldtext = f.read(len(text) + 1) - if text != oldtext: - f.seek(0) - f.truncate(0) - f.write(text) - f.close() - - -@contextmanager -def ifcontext(ifcond, *args): - """A 'with' statement context manager to wrap with start_if()/end_if() - - *args: any number of QAPIGenCCode - - Example:: - - with ifcontext(ifcond, self._genh, self._genc): - modify self._genh and self._genc ... - - Is equivalent to calling:: - - self._genh.start_if(ifcond) - self._genc.start_if(ifcond) - modify self._genh and self._genc ... - self._genh.end_if() - self._genc.end_if() - """ - for arg in args: - arg.start_if(ifcond) - yield - for arg in args: - arg.end_if() - - -class QAPIGenCCode(QAPIGen): - - def __init__(self, fname): - QAPIGen.__init__(self, fname) - self._start_if = None - - def start_if(self, ifcond): - assert self._start_if is None - self._start_if = (ifcond, self._body, self._preamble) - - def end_if(self): - assert self._start_if - self._wrap_ifcond() - self._start_if = None - - def _wrap_ifcond(self): - self._body = _wrap_ifcond(self._start_if[0], - self._start_if[1], self._body) - self._preamble = _wrap_ifcond(self._start_if[0], - self._start_if[2], self._preamble) - - def get_content(self): - assert self._start_if is None - return QAPIGen.get_content(self) - - -class QAPIGenC(QAPIGenCCode): - - def __init__(self, fname, blurb, pydoc): - QAPIGenCCode.__init__(self, fname) - self._blurb = blurb - self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc, - re.MULTILINE)) - - def _top(self): - return mcgen(''' -/* AUTOMATICALLY GENERATED, DO NOT MODIFY */ - -/* -%(blurb)s - * - * %(copyright)s - * - * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. - * See the COPYING.LIB file in the top-level directory. - */ - -''', - blurb=self._blurb, copyright=self._copyright) - - def _bottom(self): - return mcgen(''' - -/* Dummy declaration to prevent empty .o file */ -char qapi_dummy_%(name)s; -''', - name=c_fname(self.fname)) - - -class QAPIGenH(QAPIGenC): - - def _top(self): - return QAPIGenC._top(self) + guardstart(self.fname) - - def _bottom(self): - return guardend(self.fname) - - -class QAPIGenDoc(QAPIGen): - - def _top(self): - return (QAPIGen._top(self) - + '@c AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n') - - -class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor): - - def __init__(self, prefix, what, blurb, pydoc): - self._prefix = prefix - self._what = what - self._genc = QAPIGenC(self._prefix + self._what + '.c', - blurb, pydoc) - self._genh = QAPIGenH(self._prefix + self._what + '.h', - blurb, pydoc) - - def write(self, output_dir): - self._genc.write(output_dir) - self._genh.write(output_dir) - - -class QAPISchemaModularCVisitor(QAPISchemaVisitor): - - def __init__(self, prefix, what, blurb, pydoc): - self._prefix = prefix - self._what = what - self._blurb = blurb - self._pydoc = pydoc - self._genc = None - self._genh = None - self._module = {} - self._main_module = None - - @staticmethod - def _is_user_module(name): - return name and not name.startswith('./') - - @staticmethod - def _is_builtin_module(name): - return not name - - def _module_dirname(self, what, name): - if self._is_user_module(name): - return os.path.dirname(name) - return '' - - def _module_basename(self, what, name): - ret = '' if self._is_builtin_module(name) else self._prefix - if self._is_user_module(name): - basename = os.path.basename(name) - ret += what - if name != self._main_module: - ret += '-' + os.path.splitext(basename)[0] - else: - name = name[2:] if name else 'builtin' - ret += re.sub(r'-', '-' + name + '-', what) - return ret - - def _module_filename(self, what, name): - return os.path.join(self._module_dirname(what, name), - self._module_basename(what, name)) - - def _add_module(self, name, blurb): - basename = self._module_filename(self._what, name) - genc = QAPIGenC(basename + '.c', blurb, self._pydoc) - genh = QAPIGenH(basename + '.h', blurb, self._pydoc) - self._module[name] = (genc, genh) - self._set_module(name) - - def _add_user_module(self, name, blurb): - assert self._is_user_module(name) - if self._main_module is None: - self._main_module = name - self._add_module(name, blurb) - - def _add_system_module(self, name, blurb): - self._add_module(name and './' + name, blurb) - - def _set_module(self, name): - self._genc, self._genh = self._module[name] - - def write(self, output_dir, opt_builtins=False): - for name in self._module: - if self._is_builtin_module(name) and not opt_builtins: - continue - (genc, genh) = self._module[name] - genc.write(output_dir) - genh.write(output_dir) - - def _begin_user_module(self, name): - pass - - def visit_module(self, name): - if name in self._module: - self._set_module(name) - elif self._is_builtin_module(name): - # The built-in module has not been created. No code may - # be generated. - self._genc = None - self._genh = None - else: - self._add_user_module(name, self._blurb) - self._begin_user_module(name) - - def visit_include(self, name, info): - relname = os.path.relpath(self._module_filename(self._what, name), - os.path.dirname(self._genh.fname)) - self._genh.preamble_add(mcgen(''' -#include "%(relname)s.h" -''', - relname=relname)) diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py index 693cc4486b..1c5125249f 100755 --- a/scripts/qapi/doc.py +++ b/scripts/qapi/doc.py @@ -7,7 +7,8 @@ from __future__ import print_function import re -import qapi.common +from qapi.gen import QAPIGenDoc, QAPISchemaVisitor + MSG_FMT = """ @deftypefn {type} {{}} {name} @@ -216,10 +217,10 @@ def texi_entity(doc, what, ifcond, base=None, variants=None, + texi_sections(doc, ifcond)) -class QAPISchemaGenDocVisitor(qapi.common.QAPISchemaVisitor): +class QAPISchemaGenDocVisitor(QAPISchemaVisitor): def __init__(self, prefix): self._prefix = prefix - self._gen = qapi.common.QAPIGenDoc(self._prefix + 'qapi-doc.texi') + self._gen = QAPIGenDoc(self._prefix + 'qapi-doc.texi') self.cur_doc = None def write(self, output_dir): diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py new file mode 100644 index 0000000000..b9f3751bea --- /dev/null +++ b/scripts/qapi/error.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# +# QAPI error classes +# +# Copyright (c) 2017-2019 Red Hat Inc. +# +# Authors: +# Markus Armbruster <armbru@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + + +class QAPIError(Exception): + def __init__(self, info, col, msg): + Exception.__init__(self) + self.info = info + self.col = col + self.msg = msg + + def __str__(self): + loc = str(self.info) + if self.col is not None: + assert self.info.line is not None + loc += ':%s' % self.col + return loc + ': ' + self.msg + + +class QAPIParseError(QAPIError): + def __init__(self, parser, msg): + col = 1 + for ch in parser.src[parser.line_pos:parser.pos]: + if ch == '\t': + col = (col + 7) % 8 + 1 + else: + col += 1 + QAPIError.__init__(self, parser.info, col, msg) + + +class QAPISemError(QAPIError): + def __init__(self, info, msg): + QAPIError.__init__(self, info, None, msg) diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py index a716a1d27f..10fc509fa9 100644 --- a/scripts/qapi/events.py +++ b/scripts/qapi/events.py @@ -13,6 +13,8 @@ See the COPYING file in the top-level directory. """ from qapi.common import * +from qapi.gen import QAPISchemaModularCVisitor, ifcontext +from qapi.schema import QAPISchemaEnumMember from qapi.types import gen_enum, gen_enum_lookup diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py new file mode 100644 index 0000000000..67cb2c2b6c --- /dev/null +++ b/scripts/qapi/expr.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- +# +# Check (context-free) QAPI schema expression structure +# +# Copyright IBM, Corp. 2011 +# Copyright (c) 2013-2019 Red Hat Inc. +# +# Authors: +# Anthony Liguori <aliguori@us.ibm.com> +# Markus Armbruster <armbru@redhat.com> +# Eric Blake <eblake@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +import re +from collections import OrderedDict +from qapi.common import c_name +from qapi.error import QAPISemError + + +# Names must be letters, numbers, -, and _. They must start with letter, +# except for downstream extensions which must start with __RFQDN_. +# Dots are only valid in the downstream extension prefix. +valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?' + '[a-zA-Z][a-zA-Z0-9_-]*$') + + +def check_name_is_str(name, info, source): + if not isinstance(name, str): + raise QAPISemError(info, "%s requires a string name" % source) + + +def check_name_str(name, info, source, + allow_optional=False, enum_member=False, + permit_upper=False): + global valid_name + membername = name + + if allow_optional and name.startswith('*'): + membername = name[1:] + # Enum members can start with a digit, because the generated C + # code always prefixes it with the enum name + if enum_member and membername[0].isdigit(): + membername = 'D' + membername + # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' + # and 'q_obj_*' implicit type names. + if not valid_name.match(membername) or \ + c_name(membername, False).startswith('q_'): + raise QAPISemError(info, "%s has an invalid name" % source) + if not permit_upper and name.lower() != name: + raise QAPISemError( + info, "%s uses uppercase in name" % source) + assert not membername.startswith('*') + + +def check_defn_name_str(name, info, meta): + check_name_str(name, info, meta, permit_upper=True) + if name.endswith('Kind') or name.endswith('List'): + raise QAPISemError( + info, "%s name should not end in '%s'" % (meta, name[-4:])) + + +def check_keys(value, info, source, required, optional): + + def pprint(elems): + return ', '.join("'" + e + "'" for e in sorted(elems)) + + missing = set(required) - set(value) + if missing: + raise QAPISemError( + info, + "%s misses key%s %s" + % (source, 's' if len(missing) > 1 else '', + pprint(missing))) + allowed = set(required + optional) + unknown = set(value) - allowed + if unknown: + raise QAPISemError( + info, + "%s has unknown key%s %s\nValid keys are %s." + % (source, 's' if len(unknown) > 1 else '', + pprint(unknown), pprint(allowed))) + + +def check_flags(expr, info): + for key in ['gen', 'success-response']: + if key in expr and expr[key] is not False: + raise QAPISemError( + info, "flag '%s' may only use false value" % key) + for key in ['boxed', 'allow-oob', 'allow-preconfig']: + if key in expr and expr[key] is not True: + raise QAPISemError( + info, "flag '%s' may only use true value" % key) + + +def normalize_if(expr): + ifcond = expr.get('if') + if isinstance(ifcond, str): + expr['if'] = [ifcond] + + +def check_if(expr, info, source): + + def check_if_str(ifcond, info): + if not isinstance(ifcond, str): + raise QAPISemError( + info, + "'if' condition of %s must be a string or a list of strings" + % source) + if ifcond.strip() == '': + raise QAPISemError( + info, + "'if' condition '%s' of %s makes no sense" + % (ifcond, source)) + + ifcond = expr.get('if') + if ifcond is None: + return + if isinstance(ifcond, list): + if ifcond == []: + raise QAPISemError( + info, "'if' condition [] of %s is useless" % source) + for elt in ifcond: + check_if_str(elt, info) + else: + check_if_str(ifcond, info) + + +def normalize_members(members): + if isinstance(members, OrderedDict): + for key, arg in members.items(): + if isinstance(arg, dict): + continue + members[key] = {'type': arg} + + +def check_type(value, info, source, + allow_array=False, allow_dict=False): + if value is None: + return + + # Array type + if isinstance(value, list): + if not allow_array: + raise QAPISemError(info, "%s cannot be an array" % source) + if len(value) != 1 or not isinstance(value[0], str): + raise QAPISemError(info, + "%s: array type must contain single type name" % + source) + return + + # Type name + if isinstance(value, str): + return + + # Anonymous type + + if not allow_dict: + raise QAPISemError(info, "%s should be a type name" % source) + + if not isinstance(value, OrderedDict): + raise QAPISemError(info, + "%s should be an object or type name" % source) + + permit_upper = allow_dict in info.pragma.name_case_whitelist + + # value is a dictionary, check that each member is okay + for (key, arg) in value.items(): + key_source = "%s member '%s'" % (source, key) + check_name_str(key, info, key_source, + allow_optional=True, permit_upper=permit_upper) + if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): + raise QAPISemError(info, "%s uses reserved name" % key_source) + check_keys(arg, info, key_source, ['type'], ['if']) + check_if(arg, info, key_source) + normalize_if(arg) + check_type(arg['type'], info, key_source, allow_array=True) + + +def normalize_features(features): + if isinstance(features, list): + features[:] = [f if isinstance(f, dict) else {'name': f} + for f in features] + + +def normalize_enum(expr): + if isinstance(expr['data'], list): + expr['data'] = [m if isinstance(m, dict) else {'name': m} + for m in expr['data']] + + +def check_enum(expr, info): + name = expr['enum'] + members = expr['data'] + prefix = expr.get('prefix') + + if not isinstance(members, list): + raise QAPISemError(info, "'data' must be an array") + if prefix is not None and not isinstance(prefix, str): + raise QAPISemError(info, "'prefix' must be a string") + + permit_upper = name in info.pragma.name_case_whitelist + + for member in members: + source = "'data' member" + check_keys(member, info, source, ['name'], ['if']) + check_name_is_str(member['name'], info, source) + source = "%s '%s'" % (source, member['name']) + check_name_str(member['name'], info, source, + enum_member=True, permit_upper=permit_upper) + check_if(member, info, source) + normalize_if(member) + + +def check_struct(expr, info): + name = expr['struct'] + members = expr['data'] + features = expr.get('features') + + check_type(members, info, "'data'", allow_dict=name) + check_type(expr.get('base'), info, "'base'") + + if features: + if not isinstance(features, list): + raise QAPISemError(info, "'features' must be an array") + for f in features: + source = "'features' member" + assert isinstance(f, dict) + check_keys(f, info, source, ['name'], ['if']) + check_name_is_str(f['name'], info, source) + source = "%s '%s'" % (source, f['name']) + check_name_str(f['name'], info, source) + check_if(f, info, source) + normalize_if(f) + + +def check_union(expr, info): + name = expr['union'] + base = expr.get('base') + discriminator = expr.get('discriminator') + members = expr['data'] + + if discriminator is None: # simple union + if base is not None: + raise QAPISemError(info, "'base' requires 'discriminator'") + else: # flat union + check_type(base, info, "'base'", allow_dict=name) + if not base: + raise QAPISemError(info, "'discriminator' requires 'base'") + check_name_is_str(discriminator, info, "'discriminator'") + + for (key, value) in members.items(): + source = "'data' member '%s'" % key + check_name_str(key, info, source) + check_keys(value, info, source, ['type'], ['if']) + check_if(value, info, source) + normalize_if(value) + check_type(value['type'], info, source, allow_array=not base) + + +def check_alternate(expr, info): + members = expr['data'] + + if len(members) == 0: + raise QAPISemError(info, "'data' must not be empty") + for (key, value) in members.items(): + source = "'data' member '%s'" % key + check_name_str(key, info, source) + check_keys(value, info, source, ['type'], ['if']) + check_if(value, info, source) + normalize_if(value) + check_type(value['type'], info, source) + + +def check_command(expr, info): + args = expr.get('data') + rets = expr.get('returns') + boxed = expr.get('boxed', False) + + if boxed and args is None: + raise QAPISemError(info, "'boxed': true requires 'data'") + check_type(args, info, "'data'", allow_dict=not boxed) + check_type(rets, info, "'returns'", allow_array=True) + + +def check_event(expr, info): + args = expr.get('data') + boxed = expr.get('boxed', False) + + if boxed and args is None: + raise QAPISemError(info, "'boxed': true requires 'data'") + check_type(args, info, "'data'", allow_dict=not boxed) + + +def check_exprs(exprs): + for expr_elem in exprs: + expr = expr_elem['expr'] + info = expr_elem['info'] + doc = expr_elem.get('doc') + + if 'include' in expr: + continue + + if 'enum' in expr: + meta = 'enum' + elif 'union' in expr: + meta = 'union' + elif 'alternate' in expr: + meta = 'alternate' + elif 'struct' in expr: + meta = 'struct' + elif 'command' in expr: + meta = 'command' + elif 'event' in expr: + meta = 'event' + else: + raise QAPISemError(info, "expression is missing metatype") + + name = expr[meta] + check_name_is_str(name, info, "'%s'" % meta) + info.set_defn(meta, name) + check_defn_name_str(name, info, meta) + + if doc: + if doc.symbol != name: + raise QAPISemError( + info, "documentation comment is for '%s'" % doc.symbol) + doc.check_expr(expr) + elif info.pragma.doc_required: + raise QAPISemError(info, + "documentation comment required") + + if meta == 'enum': + check_keys(expr, info, meta, + ['enum', 'data'], ['if', 'prefix']) + normalize_enum(expr) + check_enum(expr, info) + elif meta == 'union': + check_keys(expr, info, meta, + ['union', 'data'], + ['base', 'discriminator', 'if']) + normalize_members(expr.get('base')) + normalize_members(expr['data']) + check_union(expr, info) + elif meta == 'alternate': + check_keys(expr, info, meta, + ['alternate', 'data'], ['if']) + normalize_members(expr['data']) + check_alternate(expr, info) + elif meta == 'struct': + check_keys(expr, info, meta, + ['struct', 'data'], ['base', 'if', 'features']) + normalize_members(expr['data']) + normalize_features(expr.get('features')) + check_struct(expr, info) + elif meta == 'command': + check_keys(expr, info, meta, + ['command'], + ['data', 'returns', 'boxed', 'if', + 'gen', 'success-response', 'allow-oob', + 'allow-preconfig']) + normalize_members(expr.get('data')) + check_command(expr, info) + elif meta == 'event': + check_keys(expr, info, meta, + ['event'], ['data', 'boxed', 'if']) + normalize_members(expr.get('data')) + check_event(expr, info) + else: + assert False, 'unexpected meta type' + + normalize_if(expr) + check_if(expr, info, meta) + check_flags(expr, info) + + return exprs diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py new file mode 100644 index 0000000000..112b6d94c5 --- /dev/null +++ b/scripts/qapi/gen.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +# +# QAPI code generation +# +# Copyright (c) 2018-2019 Red Hat Inc. +# +# Authors: +# Markus Armbruster <armbru@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + + +import errno +import os +import re +import sys +from contextlib import contextmanager + +from qapi.common import * +from qapi.schema import QAPISchemaVisitor + + +class QAPIGen(object): + + def __init__(self, fname): + self.fname = fname + self._preamble = '' + self._body = '' + + def preamble_add(self, text): + self._preamble += text + + def add(self, text): + self._body += text + + def get_content(self): + return self._top() + self._preamble + self._body + self._bottom() + + def _top(self): + return '' + + def _bottom(self): + return '' + + def write(self, output_dir): + pathname = os.path.join(output_dir, self.fname) + dir = os.path.dirname(pathname) + if dir: + try: + os.makedirs(dir) + except os.error as e: + if e.errno != errno.EEXIST: + raise + fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666) + if sys.version_info[0] >= 3: + f = open(fd, 'r+', encoding='utf-8') + else: + f = os.fdopen(fd, 'r+') + text = self.get_content() + oldtext = f.read(len(text) + 1) + if text != oldtext: + f.seek(0) + f.truncate(0) + f.write(text) + f.close() + + +def _wrap_ifcond(ifcond, before, after): + if before == after: + return after # suppress empty #if ... #endif + + assert after.startswith(before) + out = before + added = after[len(before):] + if added[0] == '\n': + out += '\n' + added = added[1:] + out += gen_if(ifcond) + out += added + out += gen_endif(ifcond) + return out + + +class QAPIGenCCode(QAPIGen): + + def __init__(self, fname): + QAPIGen.__init__(self, fname) + self._start_if = None + + def start_if(self, ifcond): + assert self._start_if is None + self._start_if = (ifcond, self._body, self._preamble) + + def end_if(self): + assert self._start_if + self._wrap_ifcond() + self._start_if = None + + def _wrap_ifcond(self): + self._body = _wrap_ifcond(self._start_if[0], + self._start_if[1], self._body) + self._preamble = _wrap_ifcond(self._start_if[0], + self._start_if[2], self._preamble) + + def get_content(self): + assert self._start_if is None + return QAPIGen.get_content(self) + + +class QAPIGenC(QAPIGenCCode): + + def __init__(self, fname, blurb, pydoc): + QAPIGenCCode.__init__(self, fname) + self._blurb = blurb + self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc, + re.MULTILINE)) + + def _top(self): + return mcgen(''' +/* AUTOMATICALLY GENERATED, DO NOT MODIFY */ + +/* +%(blurb)s + * + * %(copyright)s + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + */ + +''', + blurb=self._blurb, copyright=self._copyright) + + def _bottom(self): + return mcgen(''' + +/* Dummy declaration to prevent empty .o file */ +char qapi_dummy_%(name)s; +''', + name=c_fname(self.fname)) + + +class QAPIGenH(QAPIGenC): + + def _top(self): + return QAPIGenC._top(self) + guardstart(self.fname) + + def _bottom(self): + return guardend(self.fname) + + +@contextmanager +def ifcontext(ifcond, *args): + """A 'with' statement context manager to wrap with start_if()/end_if() + + *args: any number of QAPIGenCCode + + Example:: + + with ifcontext(ifcond, self._genh, self._genc): + modify self._genh and self._genc ... + + Is equivalent to calling:: + + self._genh.start_if(ifcond) + self._genc.start_if(ifcond) + modify self._genh and self._genc ... + self._genh.end_if() + self._genc.end_if() + """ + for arg in args: + arg.start_if(ifcond) + yield + for arg in args: + arg.end_if() + + +class QAPIGenDoc(QAPIGen): + + def _top(self): + return (QAPIGen._top(self) + + '@c AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n') + + +class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor): + + def __init__(self, prefix, what, blurb, pydoc): + self._prefix = prefix + self._what = what + self._genc = QAPIGenC(self._prefix + self._what + '.c', + blurb, pydoc) + self._genh = QAPIGenH(self._prefix + self._what + '.h', + blurb, pydoc) + + def write(self, output_dir): + self._genc.write(output_dir) + self._genh.write(output_dir) + + +class QAPISchemaModularCVisitor(QAPISchemaVisitor): + + def __init__(self, prefix, what, blurb, pydoc): + self._prefix = prefix + self._what = what + self._blurb = blurb + self._pydoc = pydoc + self._genc = None + self._genh = None + self._module = {} + self._main_module = None + + @staticmethod + def _is_user_module(name): + return name and not name.startswith('./') + + @staticmethod + def _is_builtin_module(name): + return not name + + def _module_dirname(self, what, name): + if self._is_user_module(name): + return os.path.dirname(name) + return '' + + def _module_basename(self, what, name): + ret = '' if self._is_builtin_module(name) else self._prefix + if self._is_user_module(name): + basename = os.path.basename(name) + ret += what + if name != self._main_module: + ret += '-' + os.path.splitext(basename)[0] + else: + name = name[2:] if name else 'builtin' + ret += re.sub(r'-', '-' + name + '-', what) + return ret + + def _module_filename(self, what, name): + return os.path.join(self._module_dirname(what, name), + self._module_basename(what, name)) + + def _add_module(self, name, blurb): + basename = self._module_filename(self._what, name) + genc = QAPIGenC(basename + '.c', blurb, self._pydoc) + genh = QAPIGenH(basename + '.h', blurb, self._pydoc) + self._module[name] = (genc, genh) + self._set_module(name) + + def _add_user_module(self, name, blurb): + assert self._is_user_module(name) + if self._main_module is None: + self._main_module = name + self._add_module(name, blurb) + + def _add_system_module(self, name, blurb): + self._add_module(name and './' + name, blurb) + + def _set_module(self, name): + self._genc, self._genh = self._module[name] + + def write(self, output_dir, opt_builtins=False): + for name in self._module: + if self._is_builtin_module(name) and not opt_builtins: + continue + (genc, genh) = self._module[name] + genc.write(output_dir) + genh.write(output_dir) + + def _begin_user_module(self, name): + pass + + def visit_module(self, name): + if name in self._module: + self._set_module(name) + elif self._is_builtin_module(name): + # The built-in module has not been created. No code may + # be generated. + self._genc = None + self._genh = None + else: + self._add_user_module(name, self._blurb) + self._begin_user_module(name) + + def visit_include(self, name, info): + relname = os.path.relpath(self._module_filename(self._what, name), + os.path.dirname(self._genh.fname)) + self._genh.preamble_add(mcgen(''' +#include "%(relname)s.h" +''', + relname=relname)) diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index f62cf0a2e1..4f257591de 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -10,7 +10,12 @@ This work is licensed under the terms of the GNU GPL, version 2. See the COPYING file in the top-level directory. """ +import string + from qapi.common import * +from qapi.gen import QAPISchemaMonolithicCVisitor +from qapi.schema import (QAPISchemaArrayType, QAPISchemaBuiltinType, + QAPISchemaType) def to_qlit(obj, level=0, suppress_first_indent=False): diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py new file mode 100644 index 0000000000..e800876ad1 --- /dev/null +++ b/scripts/qapi/parser.py @@ -0,0 +1,570 @@ +# -*- coding: utf-8 -*- +# +# QAPI schema parser +# +# Copyright IBM, Corp. 2011 +# Copyright (c) 2013-2019 Red Hat Inc. +# +# Authors: +# Anthony Liguori <aliguori@us.ibm.com> +# Markus Armbruster <armbru@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# Kevin Wolf <kwolf@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +import os +import re +import sys +from collections import OrderedDict + +from qapi.error import QAPIParseError, QAPISemError +from qapi.source import QAPISourceInfo + + +class QAPISchemaParser(object): + + def __init__(self, fname, previously_included=None, incl_info=None): + previously_included = previously_included or set() + previously_included.add(os.path.abspath(fname)) + + try: + if sys.version_info[0] >= 3: + fp = open(fname, 'r', encoding='utf-8') + else: + fp = open(fname, 'r') + self.src = fp.read() + except IOError as e: + raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), + "can't read %s file '%s': %s" + % ("include" if incl_info else "schema", + fname, + e.strerror)) + + if self.src == '' or self.src[-1] != '\n': + self.src += '\n' + self.cursor = 0 + self.info = QAPISourceInfo(fname, 1, incl_info) + self.line_pos = 0 + self.exprs = [] + self.docs = [] + self.accept() + cur_doc = None + + while self.tok is not None: + info = self.info + if self.tok == '#': + self.reject_expr_doc(cur_doc) + cur_doc = self.get_doc(info) + self.docs.append(cur_doc) + continue + + expr = self.get_expr(False) + if 'include' in expr: + self.reject_expr_doc(cur_doc) + if len(expr) != 1: + raise QAPISemError(info, "invalid 'include' directive") + include = expr['include'] + if not isinstance(include, str): + raise QAPISemError(info, + "value of 'include' must be a string") + incl_fname = os.path.join(os.path.dirname(fname), + include) + self.exprs.append({'expr': {'include': incl_fname}, + 'info': info}) + exprs_include = self._include(include, info, incl_fname, + previously_included) + if exprs_include: + self.exprs.extend(exprs_include.exprs) + self.docs.extend(exprs_include.docs) + elif "pragma" in expr: + self.reject_expr_doc(cur_doc) + if len(expr) != 1: + raise QAPISemError(info, "invalid 'pragma' directive") + pragma = expr['pragma'] + if not isinstance(pragma, dict): + raise QAPISemError( + info, "value of 'pragma' must be an object") + for name, value in pragma.items(): + self._pragma(name, value, info) + else: + expr_elem = {'expr': expr, + 'info': info} + if cur_doc: + if not cur_doc.symbol: + raise QAPISemError( + cur_doc.info, "definition documentation required") + expr_elem['doc'] = cur_doc + self.exprs.append(expr_elem) + cur_doc = None + self.reject_expr_doc(cur_doc) + + @staticmethod + def reject_expr_doc(doc): + if doc and doc.symbol: + raise QAPISemError( + doc.info, + "documentation for '%s' is not followed by the definition" + % doc.symbol) + + def _include(self, include, info, incl_fname, previously_included): + incl_abs_fname = os.path.abspath(incl_fname) + # catch inclusion cycle + inf = info + while inf: + if incl_abs_fname == os.path.abspath(inf.fname): + raise QAPISemError(info, "inclusion loop for %s" % include) + inf = inf.parent + + # skip multiple include of the same file + if incl_abs_fname in previously_included: + return None + + return QAPISchemaParser(incl_fname, previously_included, info) + + def _pragma(self, name, value, info): + if name == 'doc-required': + if not isinstance(value, bool): + raise QAPISemError(info, + "pragma 'doc-required' must be boolean") + info.pragma.doc_required = value + elif name == 'returns-whitelist': + if (not isinstance(value, list) + or any([not isinstance(elt, str) for elt in value])): + raise QAPISemError( + info, + "pragma returns-whitelist must be a list of strings") + info.pragma.returns_whitelist = value + elif name == 'name-case-whitelist': + if (not isinstance(value, list) + or any([not isinstance(elt, str) for elt in value])): + raise QAPISemError( + info, + "pragma name-case-whitelist must be a list of strings") + info.pragma.name_case_whitelist = value + else: + raise QAPISemError(info, "unknown pragma '%s'" % name) + + def accept(self, skip_comment=True): + while True: + self.tok = self.src[self.cursor] + self.pos = self.cursor + self.cursor += 1 + self.val = None + + if self.tok == '#': + if self.src[self.cursor] == '#': + # Start of doc comment + skip_comment = False + self.cursor = self.src.find('\n', self.cursor) + if not skip_comment: + self.val = self.src[self.pos:self.cursor] + return + elif self.tok in '{}:,[]': + return + elif self.tok == "'": + # Note: we accept only printable ASCII + string = '' + esc = False + while True: + ch = self.src[self.cursor] + self.cursor += 1 + if ch == '\n': + raise QAPIParseError(self, "missing terminating \"'\"") + if esc: + # Note: we recognize only \\ because we have + # no use for funny characters in strings + if ch != '\\': + raise QAPIParseError(self, + "unknown escape \\%s" % ch) + esc = False + elif ch == '\\': + esc = True + continue + elif ch == "'": + self.val = string + return + if ord(ch) < 32 or ord(ch) >= 127: + raise QAPIParseError( + self, "funny character in string") + string += ch + elif self.src.startswith('true', self.pos): + self.val = True + self.cursor += 3 + return + elif self.src.startswith('false', self.pos): + self.val = False + self.cursor += 4 + return + elif self.tok == '\n': + if self.cursor == len(self.src): + self.tok = None + return + self.info = self.info.next_line() + self.line_pos = self.cursor + elif not self.tok.isspace(): + # Show up to next structural, whitespace or quote + # character + match = re.match('[^[\\]{}:,\\s\'"]+', + self.src[self.cursor-1:]) + raise QAPIParseError(self, "stray '%s'" % match.group(0)) + + def get_members(self): + expr = OrderedDict() + if self.tok == '}': + self.accept() + return expr + if self.tok != "'": + raise QAPIParseError(self, "expected string or '}'") + while True: + key = self.val + self.accept() + if self.tok != ':': + raise QAPIParseError(self, "expected ':'") + self.accept() + if key in expr: + raise QAPIParseError(self, "duplicate key '%s'" % key) + expr[key] = self.get_expr(True) + if self.tok == '}': + self.accept() + return expr + if self.tok != ',': + raise QAPIParseError(self, "expected ',' or '}'") + self.accept() + if self.tok != "'": + raise QAPIParseError(self, "expected string") + + def get_values(self): + expr = [] + if self.tok == ']': + self.accept() + return expr + if self.tok not in "{['tfn": + raise QAPIParseError( + self, "expected '{', '[', ']', string, boolean or 'null'") + while True: + expr.append(self.get_expr(True)) + if self.tok == ']': + self.accept() + return expr + if self.tok != ',': + raise QAPIParseError(self, "expected ',' or ']'") + self.accept() + + def get_expr(self, nested): + if self.tok != '{' and not nested: + raise QAPIParseError(self, "expected '{'") + if self.tok == '{': + self.accept() + expr = self.get_members() + elif self.tok == '[': + self.accept() + expr = self.get_values() + elif self.tok in "'tfn": + expr = self.val + self.accept() + else: + raise QAPIParseError( + self, "expected '{', '[', string, boolean or 'null'") + return expr + + def get_doc(self, info): + if self.val != '##': + raise QAPIParseError( + self, "junk after '##' at start of documentation comment") + + doc = QAPIDoc(self, info) + self.accept(False) + while self.tok == '#': + if self.val.startswith('##'): + # End of doc comment + if self.val != '##': + raise QAPIParseError( + self, + "junk after '##' at end of documentation comment") + doc.end_comment() + self.accept() + return doc + else: + doc.append(self.val) + self.accept(False) + + raise QAPIParseError(self, "documentation comment must end with '##'") + + +class QAPIDoc(object): + """ + A documentation comment block, either definition or free-form + + Definition documentation blocks consist of + + * a body section: one line naming the definition, followed by an + overview (any number of lines) + + * argument sections: a description of each argument (for commands + and events) or member (for structs, unions and alternates) + + * features sections: a description of each feature flag + + * additional (non-argument) sections, possibly tagged + + Free-form documentation blocks consist only of a body section. + """ + + class Section(object): + def __init__(self, name=None): + # optional section name (argument/member or section name) + self.name = name + # the list of lines for this section + self.text = '' + + def append(self, line): + self.text += line.rstrip() + '\n' + + class ArgSection(Section): + def __init__(self, name): + QAPIDoc.Section.__init__(self, name) + self.member = None + + def connect(self, member): + self.member = member + + def __init__(self, parser, info): + # self._parser is used to report errors with QAPIParseError. The + # resulting error position depends on the state of the parser. + # It happens to be the beginning of the comment. More or less + # servicable, but action at a distance. + self._parser = parser + self.info = info + self.symbol = None + self.body = QAPIDoc.Section() + # dict mapping parameter name to ArgSection + self.args = OrderedDict() + self.features = OrderedDict() + # a list of Section + self.sections = [] + # the current section + self._section = self.body + self._append_line = self._append_body_line + + def has_section(self, name): + """Return True if we have a section with this name.""" + for i in self.sections: + if i.name == name: + return True + return False + + def append(self, line): + """ + Parse a comment line and add it to the documentation. + + The way that the line is dealt with depends on which part of + the documentation we're parsing right now: + * The body section: ._append_line is ._append_body_line + * An argument section: ._append_line is ._append_args_line + * A features section: ._append_line is ._append_features_line + * An additional section: ._append_line is ._append_various_line + """ + line = line[1:] + if not line: + self._append_freeform(line) + return + + if line[0] != ' ': + raise QAPIParseError(self._parser, "missing space after #") + line = line[1:] + self._append_line(line) + + def end_comment(self): + self._end_section() + + @staticmethod + def _is_section_tag(name): + return name in ('Returns:', 'Since:', + # those are often singular or plural + 'Note:', 'Notes:', + 'Example:', 'Examples:', + 'TODO:') + + def _append_body_line(self, line): + """ + Process a line of documentation text in the body section. + + If this a symbol line and it is the section's first line, this + is a definition documentation block for that symbol. + + If it's a definition documentation block, another symbol line + begins the argument section for the argument named by it, and + a section tag begins an additional section. Start that + section and append the line to it. + + Else, append the line to the current section. + """ + name = line.split(' ', 1)[0] + # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't + # recognized, and get silently treated as ordinary text + if not self.symbol and not self.body.text and line.startswith('@'): + if not line.endswith(':'): + raise QAPIParseError(self._parser, "line should end with ':'") + self.symbol = line[1:-1] + # FIXME invalid names other than the empty string aren't flagged + if not self.symbol: + raise QAPIParseError(self._parser, "invalid name") + elif self.symbol: + # This is a definition documentation block + if name.startswith('@') and name.endswith(':'): + self._append_line = self._append_args_line + self._append_args_line(line) + elif line == 'Features:': + self._append_line = self._append_features_line + elif self._is_section_tag(name): + self._append_line = self._append_various_line + self._append_various_line(line) + else: + self._append_freeform(line.strip()) + else: + # This is a free-form documentation block + self._append_freeform(line.strip()) + + def _append_args_line(self, line): + """ + Process a line of documentation text in an argument section. + + A symbol line begins the next argument section, a section tag + section or a non-indented line after a blank line begins an + additional section. Start that section and append the line to + it. + + Else, append the line to the current section. + + """ + name = line.split(' ', 1)[0] + + if name.startswith('@') and name.endswith(':'): + line = line[len(name)+1:] + self._start_args_section(name[1:-1]) + elif self._is_section_tag(name): + self._append_line = self._append_various_line + self._append_various_line(line) + return + elif (self._section.text.endswith('\n\n') + and line and not line[0].isspace()): + if line == 'Features:': + self._append_line = self._append_features_line + else: + self._start_section() + self._append_line = self._append_various_line + self._append_various_line(line) + return + + self._append_freeform(line.strip()) + + def _append_features_line(self, line): + name = line.split(' ', 1)[0] + + if name.startswith('@') and name.endswith(':'): + line = line[len(name)+1:] + self._start_features_section(name[1:-1]) + elif self._is_section_tag(name): + self._append_line = self._append_various_line + self._append_various_line(line) + return + elif (self._section.text.endswith('\n\n') + and line and not line[0].isspace()): + self._start_section() + self._append_line = self._append_various_line + self._append_various_line(line) + return + + self._append_freeform(line.strip()) + + def _append_various_line(self, line): + """ + Process a line of documentation text in an additional section. + + A symbol line is an error. + + A section tag begins an additional section. Start that + section and append the line to it. + + Else, append the line to the current section. + """ + name = line.split(' ', 1)[0] + + if name.startswith('@') and name.endswith(':'): + raise QAPIParseError(self._parser, + "'%s' can't follow '%s' section" + % (name, self.sections[0].name)) + elif self._is_section_tag(name): + line = line[len(name)+1:] + self._start_section(name[:-1]) + + if (not self._section.name or + not self._section.name.startswith('Example')): + line = line.strip() + + self._append_freeform(line) + + def _start_symbol_section(self, symbols_dict, name): + # FIXME invalid names other than the empty string aren't flagged + if not name: + raise QAPIParseError(self._parser, "invalid parameter name") + if name in symbols_dict: + raise QAPIParseError(self._parser, + "'%s' parameter name duplicated" % name) + assert not self.sections + self._end_section() + self._section = QAPIDoc.ArgSection(name) + symbols_dict[name] = self._section + + def _start_args_section(self, name): + self._start_symbol_section(self.args, name) + + def _start_features_section(self, name): + self._start_symbol_section(self.features, name) + + def _start_section(self, name=None): + if name in ('Returns', 'Since') and self.has_section(name): + raise QAPIParseError(self._parser, + "duplicated '%s' section" % name) + self._end_section() + self._section = QAPIDoc.Section(name) + self.sections.append(self._section) + + def _end_section(self): + if self._section: + text = self._section.text = self._section.text.strip() + if self._section.name and (not text or text.isspace()): + raise QAPIParseError( + self._parser, + "empty doc section '%s'" % self._section.name) + self._section = None + + def _append_freeform(self, line): + match = re.match(r'(@\S+:)', line) + if match: + raise QAPIParseError(self._parser, + "'%s' not allowed in free-form documentation" + % match.group(1)) + self._section.append(line) + + def connect_member(self, member): + if member.name not in self.args: + # Undocumented TODO outlaw + self.args[member.name] = QAPIDoc.ArgSection(member.name) + self.args[member.name].connect(member) + + def check_expr(self, expr): + if self.has_section('Returns') and 'command' not in expr: + raise QAPISemError(self.info, + "'Returns:' is only valid for commands") + + def check(self): + bogus = [name for name, section in self.args.items() + if not section.member] + if bogus: + raise QAPISemError( + self.info, + "the following documented members are not in " + "the declaration: %s" % ", ".join(bogus)) diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py new file mode 100644 index 0000000000..2913a0fef0 --- /dev/null +++ b/scripts/qapi/schema.py @@ -0,0 +1,1043 @@ +# -*- coding: utf-8 -*- +# +# QAPI schema internal representation +# +# Copyright (c) 2015-2019 Red Hat Inc. +# +# Authors: +# Markus Armbruster <armbru@redhat.com> +# Eric Blake <eblake@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +# TODO catching name collisions in generated code would be nice + +import os +import re +from collections import OrderedDict + +from qapi.common import c_name, pointer_suffix +from qapi.error import QAPIError, QAPIParseError, QAPISemError +from qapi.expr import check_exprs +from qapi.parser import QAPISchemaParser + + +class QAPISchemaEntity(object): + meta = None + + def __init__(self, name, info, doc, ifcond=None): + assert name is None or isinstance(name, str) + self.name = name + self._module = None + # For explicitly defined entities, info points to the (explicit) + # definition. For builtins (and their arrays), info is None. + # For implicitly defined entities, info points to a place that + # triggered the implicit definition (there may be more than one + # such place). + self.info = info + self.doc = doc + self._ifcond = ifcond or [] + self._checked = False + + def c_name(self): + return c_name(self.name) + + def check(self, schema): + assert not self._checked + if self.info: + self._module = os.path.relpath(self.info.fname, + os.path.dirname(schema.fname)) + self._checked = True + + @property + def ifcond(self): + assert self._checked + return self._ifcond + + @property + def module(self): + assert self._checked + return self._module + + def is_implicit(self): + return not self.info + + def visit(self, visitor): + assert self._checked + + def describe(self): + assert self.meta + return "%s '%s'" % (self.meta, self.name) + + +class QAPISchemaVisitor(object): + def visit_begin(self, schema): + pass + + def visit_end(self): + pass + + def visit_module(self, fname): + pass + + def visit_needed(self, entity): + # Default to visiting everything + return True + + def visit_include(self, fname, info): + pass + + def visit_builtin_type(self, name, info, json_type): + pass + + def visit_enum_type(self, name, info, ifcond, members, prefix): + pass + + def visit_array_type(self, name, info, ifcond, element_type): + pass + + def visit_object_type(self, name, info, ifcond, base, members, variants, + features): + pass + + def visit_object_type_flat(self, name, info, ifcond, members, variants, + features): + pass + + def visit_alternate_type(self, name, info, ifcond, variants): + pass + + def visit_command(self, name, info, ifcond, arg_type, ret_type, gen, + success_response, boxed, allow_oob, allow_preconfig): + pass + + def visit_event(self, name, info, ifcond, arg_type, boxed): + pass + + +class QAPISchemaInclude(QAPISchemaEntity): + + def __init__(self, fname, info): + QAPISchemaEntity.__init__(self, None, info, None) + self.fname = fname + + def visit(self, visitor): + QAPISchemaEntity.visit(self, visitor) + visitor.visit_include(self.fname, self.info) + + +class QAPISchemaType(QAPISchemaEntity): + # Return the C type for common use. + # For the types we commonly box, this is a pointer type. + def c_type(self): + pass + + # Return the C type to be used in a parameter list. + def c_param_type(self): + return self.c_type() + + # Return the C type to be used where we suppress boxing. + def c_unboxed_type(self): + return self.c_type() + + def json_type(self): + pass + + def alternate_qtype(self): + json2qtype = { + 'null': 'QTYPE_QNULL', + 'string': 'QTYPE_QSTRING', + 'number': 'QTYPE_QNUM', + 'int': 'QTYPE_QNUM', + 'boolean': 'QTYPE_QBOOL', + 'object': 'QTYPE_QDICT' + } + return json2qtype.get(self.json_type()) + + def doc_type(self): + if self.is_implicit(): + return None + return self.name + + def describe(self): + assert self.meta + return "%s type '%s'" % (self.meta, self.name) + + +class QAPISchemaBuiltinType(QAPISchemaType): + meta = 'built-in' + + def __init__(self, name, json_type, c_type): + QAPISchemaType.__init__(self, name, None, None) + assert not c_type or isinstance(c_type, str) + assert json_type in ('string', 'number', 'int', 'boolean', 'null', + 'value') + self._json_type_name = json_type + self._c_type_name = c_type + + def c_name(self): + return self.name + + def c_type(self): + return self._c_type_name + + def c_param_type(self): + if self.name == 'str': + return 'const ' + self._c_type_name + return self._c_type_name + + def json_type(self): + return self._json_type_name + + def doc_type(self): + return self.json_type() + + def visit(self, visitor): + QAPISchemaType.visit(self, visitor) + visitor.visit_builtin_type(self.name, self.info, self.json_type()) + + +class QAPISchemaEnumType(QAPISchemaType): + meta = 'enum' + + def __init__(self, name, info, doc, ifcond, members, prefix): + QAPISchemaType.__init__(self, name, info, doc, ifcond) + for m in members: + assert isinstance(m, QAPISchemaEnumMember) + m.set_defined_in(name) + assert prefix is None or isinstance(prefix, str) + self.members = members + self.prefix = prefix + + def check(self, schema): + QAPISchemaType.check(self, schema) + seen = {} + for m in self.members: + m.check_clash(self.info, seen) + if self.doc: + self.doc.connect_member(m) + + def is_implicit(self): + # See QAPISchema._make_implicit_enum_type() and ._def_predefineds() + return self.name.endswith('Kind') or self.name == 'QType' + + def c_type(self): + return c_name(self.name) + + def member_names(self): + return [m.name for m in self.members] + + def json_type(self): + return 'string' + + def visit(self, visitor): + QAPISchemaType.visit(self, visitor) + visitor.visit_enum_type(self.name, self.info, self.ifcond, + self.members, self.prefix) + + +class QAPISchemaArrayType(QAPISchemaType): + meta = 'array' + + def __init__(self, name, info, element_type): + QAPISchemaType.__init__(self, name, info, None, None) + assert isinstance(element_type, str) + self._element_type_name = element_type + self.element_type = None + + def check(self, schema): + QAPISchemaType.check(self, schema) + self.element_type = schema.resolve_type( + self._element_type_name, self.info, + self.info and self.info.defn_meta) + assert not isinstance(self.element_type, QAPISchemaArrayType) + + @property + def ifcond(self): + assert self._checked + return self.element_type.ifcond + + @property + def module(self): + assert self._checked + return self.element_type.module + + def is_implicit(self): + return True + + def c_type(self): + return c_name(self.name) + pointer_suffix + + def json_type(self): + return 'array' + + def doc_type(self): + elt_doc_type = self.element_type.doc_type() + if not elt_doc_type: + return None + return 'array of ' + elt_doc_type + + def visit(self, visitor): + QAPISchemaType.visit(self, visitor) + visitor.visit_array_type(self.name, self.info, self.ifcond, + self.element_type) + + def describe(self): + assert self.meta + return "%s type ['%s']" % (self.meta, self._element_type_name) + + +class QAPISchemaObjectType(QAPISchemaType): + def __init__(self, name, info, doc, ifcond, + base, local_members, variants, features): + # struct has local_members, optional base, and no variants + # flat union has base, variants, and no local_members + # simple union has local_members, variants, and no base + QAPISchemaType.__init__(self, name, info, doc, ifcond) + self.meta = 'union' if variants else 'struct' + assert base is None or isinstance(base, str) + for m in local_members: + assert isinstance(m, QAPISchemaObjectTypeMember) + m.set_defined_in(name) + if variants is not None: + assert isinstance(variants, QAPISchemaObjectTypeVariants) + variants.set_defined_in(name) + for f in features: + assert isinstance(f, QAPISchemaFeature) + f.set_defined_in(name) + self._base_name = base + self.base = None + self.local_members = local_members + self.variants = variants + self.members = None + self.features = features + + def check(self, schema): + # This calls another type T's .check() exactly when the C + # struct emitted by gen_object() contains that T's C struct + # (pointers don't count). + if self.members is not None: + # A previous .check() completed: nothing to do + return + if self._checked: + # Recursed: C struct contains itself + raise QAPISemError(self.info, + "object %s contains itself" % self.name) + + QAPISchemaType.check(self, schema) + assert self._checked and self.members is None + + seen = OrderedDict() + if self._base_name: + self.base = schema.resolve_type(self._base_name, self.info, + "'base'") + if (not isinstance(self.base, QAPISchemaObjectType) + or self.base.variants): + raise QAPISemError( + self.info, + "'base' requires a struct type, %s isn't" + % self.base.describe()) + self.base.check(schema) + self.base.check_clash(self.info, seen) + for m in self.local_members: + m.check(schema) + m.check_clash(self.info, seen) + if self.doc: + self.doc.connect_member(m) + members = seen.values() + + if self.variants: + self.variants.check(schema, seen) + self.variants.check_clash(self.info, seen) + + # Features are in a name space separate from members + seen = {} + for f in self.features: + f.check_clash(self.info, seen) + + if self.doc: + self.doc.check() + + self.members = members # mark completed + + # Check that the members of this type do not cause duplicate JSON members, + # and update seen to track the members seen so far. Report any errors + # on behalf of info, which is not necessarily self.info + def check_clash(self, info, seen): + assert self._checked + assert not self.variants # not implemented + for m in self.members: + m.check_clash(info, seen) + + @property + def ifcond(self): + assert self._checked + if isinstance(self._ifcond, QAPISchemaType): + # Simple union wrapper type inherits from wrapped type; + # see _make_implicit_object_type() + return self._ifcond.ifcond + return self._ifcond + + def is_implicit(self): + # See QAPISchema._make_implicit_object_type(), as well as + # _def_predefineds() + return self.name.startswith('q_') + + def is_empty(self): + assert self.members is not None + return not self.members and not self.variants + + def c_name(self): + assert self.name != 'q_empty' + return QAPISchemaType.c_name(self) + + def c_type(self): + assert not self.is_implicit() + return c_name(self.name) + pointer_suffix + + def c_unboxed_type(self): + return c_name(self.name) + + def json_type(self): + return 'object' + + def visit(self, visitor): + QAPISchemaType.visit(self, visitor) + visitor.visit_object_type(self.name, self.info, self.ifcond, + self.base, self.local_members, self.variants, + self.features) + visitor.visit_object_type_flat(self.name, self.info, self.ifcond, + self.members, self.variants, + self.features) + + +class QAPISchemaMember(object): + """ Represents object members, enum members and features """ + role = 'member' + + def __init__(self, name, info, ifcond=None): + assert isinstance(name, str) + self.name = name + self.info = info + self.ifcond = ifcond or [] + self.defined_in = None + + def set_defined_in(self, name): + assert not self.defined_in + self.defined_in = name + + def check_clash(self, info, seen): + cname = c_name(self.name) + if cname in seen: + raise QAPISemError( + info, + "%s collides with %s" + % (self.describe(info), seen[cname].describe(info))) + seen[cname] = self + + def describe(self, info): + role = self.role + defined_in = self.defined_in + assert defined_in + + if defined_in.startswith('q_obj_'): + # See QAPISchema._make_implicit_object_type() - reverse the + # mapping there to create a nice human-readable description + defined_in = defined_in[6:] + if defined_in.endswith('-arg'): + # Implicit type created for a command's dict 'data' + assert role == 'member' + role = 'parameter' + elif defined_in.endswith('-base'): + # Implicit type created for a flat union's dict 'base' + role = 'base ' + role + else: + # Implicit type created for a simple union's branch + assert defined_in.endswith('-wrapper') + # Unreachable and not implemented + assert False + elif defined_in.endswith('Kind'): + # See QAPISchema._make_implicit_enum_type() + # Implicit enum created for simple union's branches + assert role == 'value' + role = 'branch' + elif defined_in != info.defn_name: + return "%s '%s' of type '%s'" % (role, self.name, defined_in) + return "%s '%s'" % (role, self.name) + + +class QAPISchemaEnumMember(QAPISchemaMember): + role = 'value' + + +class QAPISchemaFeature(QAPISchemaMember): + role = 'feature' + + +class QAPISchemaObjectTypeMember(QAPISchemaMember): + def __init__(self, name, info, typ, optional, ifcond=None): + QAPISchemaMember.__init__(self, name, info, ifcond) + assert isinstance(typ, str) + assert isinstance(optional, bool) + self._type_name = typ + self.type = None + self.optional = optional + + def check(self, schema): + assert self.defined_in + self.type = schema.resolve_type(self._type_name, self.info, + self.describe) + + +class QAPISchemaObjectTypeVariants(object): + def __init__(self, tag_name, info, tag_member, variants): + # Flat unions pass tag_name but not tag_member. + # Simple unions and alternates pass tag_member but not tag_name. + # After check(), tag_member is always set, and tag_name remains + # a reliable witness of being used by a flat union. + assert bool(tag_member) != bool(tag_name) + assert (isinstance(tag_name, str) or + isinstance(tag_member, QAPISchemaObjectTypeMember)) + for v in variants: + assert isinstance(v, QAPISchemaObjectTypeVariant) + self._tag_name = tag_name + self.info = info + self.tag_member = tag_member + self.variants = variants + + def set_defined_in(self, name): + for v in self.variants: + v.set_defined_in(name) + + def check(self, schema, seen): + if not self.tag_member: # flat union + self.tag_member = seen.get(c_name(self._tag_name)) + base = "'base'" + # Pointing to the base type when not implicit would be + # nice, but we don't know it here + if not self.tag_member or self._tag_name != self.tag_member.name: + raise QAPISemError( + self.info, + "discriminator '%s' is not a member of %s" + % (self._tag_name, base)) + # Here we do: + base_type = schema.lookup_type(self.tag_member.defined_in) + assert base_type + if not base_type.is_implicit(): + base = "base type '%s'" % self.tag_member.defined_in + if not isinstance(self.tag_member.type, QAPISchemaEnumType): + raise QAPISemError( + self.info, + "discriminator member '%s' of %s must be of enum type" + % (self._tag_name, base)) + if self.tag_member.optional: + raise QAPISemError( + self.info, + "discriminator member '%s' of %s must not be optional" + % (self._tag_name, base)) + if self.tag_member.ifcond: + raise QAPISemError( + self.info, + "discriminator member '%s' of %s must not be conditional" + % (self._tag_name, base)) + else: # simple union + assert isinstance(self.tag_member.type, QAPISchemaEnumType) + assert not self.tag_member.optional + assert self.tag_member.ifcond == [] + if self._tag_name: # flat union + # branches that are not explicitly covered get an empty type + cases = set([v.name for v in self.variants]) + for m in self.tag_member.type.members: + if m.name not in cases: + v = QAPISchemaObjectTypeVariant(m.name, self.info, + 'q_empty', m.ifcond) + v.set_defined_in(self.tag_member.defined_in) + self.variants.append(v) + if not self.variants: + raise QAPISemError(self.info, "union has no branches") + for v in self.variants: + v.check(schema) + # Union names must match enum values; alternate names are + # checked separately. Use 'seen' to tell the two apart. + if seen: + if v.name not in self.tag_member.type.member_names(): + raise QAPISemError( + self.info, + "branch '%s' is not a value of %s" + % (v.name, self.tag_member.type.describe())) + if (not isinstance(v.type, QAPISchemaObjectType) + or v.type.variants): + raise QAPISemError( + self.info, + "%s cannot use %s" + % (v.describe(self.info), v.type.describe())) + v.type.check(schema) + + def check_clash(self, info, seen): + for v in self.variants: + # Reset seen map for each variant, since qapi names from one + # branch do not affect another branch + v.type.check_clash(info, dict(seen)) + + +class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember): + role = 'branch' + + def __init__(self, name, info, typ, ifcond=None): + QAPISchemaObjectTypeMember.__init__(self, name, info, typ, + False, ifcond) + + +class QAPISchemaAlternateType(QAPISchemaType): + meta = 'alternate' + + def __init__(self, name, info, doc, ifcond, variants): + QAPISchemaType.__init__(self, name, info, doc, ifcond) + assert isinstance(variants, QAPISchemaObjectTypeVariants) + assert variants.tag_member + variants.set_defined_in(name) + variants.tag_member.set_defined_in(self.name) + self.variants = variants + + def check(self, schema): + QAPISchemaType.check(self, schema) + self.variants.tag_member.check(schema) + # Not calling self.variants.check_clash(), because there's nothing + # to clash with + self.variants.check(schema, {}) + # Alternate branch names have no relation to the tag enum values; + # so we have to check for potential name collisions ourselves. + seen = {} + types_seen = {} + for v in self.variants.variants: + v.check_clash(self.info, seen) + qtype = v.type.alternate_qtype() + if not qtype: + raise QAPISemError( + self.info, + "%s cannot use %s" + % (v.describe(self.info), v.type.describe())) + conflicting = set([qtype]) + if qtype == 'QTYPE_QSTRING': + if isinstance(v.type, QAPISchemaEnumType): + for m in v.type.members: + if m.name in ['on', 'off']: + conflicting.add('QTYPE_QBOOL') + if re.match(r'[-+0-9.]', m.name): + # lazy, could be tightened + conflicting.add('QTYPE_QNUM') + else: + conflicting.add('QTYPE_QNUM') + conflicting.add('QTYPE_QBOOL') + for qt in conflicting: + if qt in types_seen: + raise QAPISemError( + self.info, + "%s can't be distinguished from '%s'" + % (v.describe(self.info), types_seen[qt])) + types_seen[qt] = v.name + if self.doc: + self.doc.connect_member(v) + if self.doc: + self.doc.check() + + def c_type(self): + return c_name(self.name) + pointer_suffix + + def json_type(self): + return 'value' + + def visit(self, visitor): + QAPISchemaType.visit(self, visitor) + visitor.visit_alternate_type(self.name, self.info, self.ifcond, + self.variants) + + +class QAPISchemaCommand(QAPISchemaEntity): + meta = 'command' + + def __init__(self, name, info, doc, ifcond, arg_type, ret_type, + gen, success_response, boxed, allow_oob, allow_preconfig): + QAPISchemaEntity.__init__(self, name, info, doc, ifcond) + assert not arg_type or isinstance(arg_type, str) + assert not ret_type or isinstance(ret_type, str) + self._arg_type_name = arg_type + self.arg_type = None + self._ret_type_name = ret_type + self.ret_type = None + self.gen = gen + self.success_response = success_response + self.boxed = boxed + self.allow_oob = allow_oob + self.allow_preconfig = allow_preconfig + + def check(self, schema): + QAPISchemaEntity.check(self, schema) + if self._arg_type_name: + self.arg_type = schema.resolve_type( + self._arg_type_name, self.info, "command's 'data'") + if not isinstance(self.arg_type, QAPISchemaObjectType): + raise QAPISemError( + self.info, + "command's 'data' cannot take %s" + % self.arg_type.describe()) + if self.arg_type.variants and not self.boxed: + raise QAPISemError( + self.info, + "command's 'data' can take %s only with 'boxed': true" + % self.arg_type.describe()) + if self._ret_type_name: + self.ret_type = schema.resolve_type( + self._ret_type_name, self.info, "command's 'returns'") + if self.name not in self.info.pragma.returns_whitelist: + if not (isinstance(self.ret_type, QAPISchemaObjectType) + or (isinstance(self.ret_type, QAPISchemaArrayType) + and isinstance(self.ret_type.element_type, + QAPISchemaObjectType))): + raise QAPISemError( + self.info, + "command's 'returns' cannot take %s" + % self.ret_type.describe()) + + def visit(self, visitor): + QAPISchemaEntity.visit(self, visitor) + visitor.visit_command(self.name, self.info, self.ifcond, + self.arg_type, self.ret_type, + self.gen, self.success_response, + self.boxed, self.allow_oob, + self.allow_preconfig) + + +class QAPISchemaEvent(QAPISchemaEntity): + meta = 'event' + + def __init__(self, name, info, doc, ifcond, arg_type, boxed): + QAPISchemaEntity.__init__(self, name, info, doc, ifcond) + assert not arg_type or isinstance(arg_type, str) + self._arg_type_name = arg_type + self.arg_type = None + self.boxed = boxed + + def check(self, schema): + QAPISchemaEntity.check(self, schema) + if self._arg_type_name: + self.arg_type = schema.resolve_type( + self._arg_type_name, self.info, "event's 'data'") + if not isinstance(self.arg_type, QAPISchemaObjectType): + raise QAPISemError( + self.info, + "event's 'data' cannot take %s" + % self.arg_type.describe()) + if self.arg_type.variants and not self.boxed: + raise QAPISemError( + self.info, + "event's 'data' can take %s only with 'boxed': true" + % self.arg_type.describe()) + + def visit(self, visitor): + QAPISchemaEntity.visit(self, visitor) + visitor.visit_event(self.name, self.info, self.ifcond, + self.arg_type, self.boxed) + + +class QAPISchema(object): + def __init__(self, fname): + self.fname = fname + parser = QAPISchemaParser(fname) + exprs = check_exprs(parser.exprs) + self.docs = parser.docs + self._entity_list = [] + self._entity_dict = {} + self._predefining = True + self._def_predefineds() + self._predefining = False + self._def_exprs(exprs) + self.check() + + def _def_entity(self, ent): + # Only the predefined types are allowed to not have info + assert ent.info or self._predefining + self._entity_list.append(ent) + if ent.name is None: + return + # TODO reject names that differ only in '_' vs. '.' vs. '-', + # because they're liable to clash in generated C. + other_ent = self._entity_dict.get(ent.name) + if other_ent: + if other_ent.info: + where = QAPIError(other_ent.info, None, "previous definition") + raise QAPISemError( + ent.info, + "'%s' is already defined\n%s" % (ent.name, where)) + raise QAPISemError( + ent.info, "%s is already defined" % other_ent.describe()) + self._entity_dict[ent.name] = ent + + def lookup_entity(self, name, typ=None): + ent = self._entity_dict.get(name) + if typ and not isinstance(ent, typ): + return None + return ent + + def lookup_type(self, name): + return self.lookup_entity(name, QAPISchemaType) + + def resolve_type(self, name, info, what): + typ = self.lookup_type(name) + if not typ: + if callable(what): + what = what(info) + raise QAPISemError( + info, "%s uses unknown type '%s'" % (what, name)) + return typ + + def _def_include(self, expr, info, doc): + include = expr['include'] + assert doc is None + main_info = info + while main_info.parent: + main_info = main_info.parent + fname = os.path.relpath(include, os.path.dirname(main_info.fname)) + self._def_entity(QAPISchemaInclude(fname, info)) + + def _def_builtin_type(self, name, json_type, c_type): + self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type)) + # Instantiating only the arrays that are actually used would + # be nice, but we can't as long as their generated code + # (qapi-builtin-types.[ch]) may be shared by some other + # schema. + self._make_array_type(name, None) + + def _def_predefineds(self): + for t in [('str', 'string', 'char' + pointer_suffix), + ('number', 'number', 'double'), + ('int', 'int', 'int64_t'), + ('int8', 'int', 'int8_t'), + ('int16', 'int', 'int16_t'), + ('int32', 'int', 'int32_t'), + ('int64', 'int', 'int64_t'), + ('uint8', 'int', 'uint8_t'), + ('uint16', 'int', 'uint16_t'), + ('uint32', 'int', 'uint32_t'), + ('uint64', 'int', 'uint64_t'), + ('size', 'int', 'uint64_t'), + ('bool', 'boolean', 'bool'), + ('any', 'value', 'QObject' + pointer_suffix), + ('null', 'null', 'QNull' + pointer_suffix)]: + self._def_builtin_type(*t) + self.the_empty_object_type = QAPISchemaObjectType( + 'q_empty', None, None, None, None, [], None, []) + self._def_entity(self.the_empty_object_type) + + qtypes = ['none', 'qnull', 'qnum', 'qstring', 'qdict', 'qlist', + 'qbool'] + qtype_values = self._make_enum_members( + [{'name': n} for n in qtypes], None) + + self._def_entity(QAPISchemaEnumType('QType', None, None, None, + qtype_values, 'QTYPE')) + + def _make_features(self, features, info): + return [QAPISchemaFeature(f['name'], info, f.get('if')) + for f in features] + + def _make_enum_members(self, values, info): + return [QAPISchemaEnumMember(v['name'], info, v.get('if')) + for v in values] + + def _make_implicit_enum_type(self, name, info, ifcond, values): + # See also QAPISchemaObjectTypeMember.describe() + name = name + 'Kind' # reserved by check_defn_name_str() + self._def_entity(QAPISchemaEnumType( + name, info, None, ifcond, self._make_enum_members(values, info), + None)) + return name + + def _make_array_type(self, element_type, info): + name = element_type + 'List' # reserved by check_defn_name_str() + if not self.lookup_type(name): + self._def_entity(QAPISchemaArrayType(name, info, element_type)) + return name + + def _make_implicit_object_type(self, name, info, doc, ifcond, + role, members): + if not members: + return None + # See also QAPISchemaObjectTypeMember.describe() + name = 'q_obj_%s-%s' % (name, role) + typ = self.lookup_entity(name, QAPISchemaObjectType) + if typ: + # The implicit object type has multiple users. This can + # happen only for simple unions' implicit wrapper types. + # Its ifcond should be the disjunction of its user's + # ifconds. Not implemented. Instead, we always pass the + # wrapped type's ifcond, which is trivially the same for all + # users. It's also necessary for the wrapper to compile. + # But it's not tight: the disjunction need not imply it. We + # may end up compiling useless wrapper types. + # TODO kill simple unions or implement the disjunction + assert (ifcond or []) == typ._ifcond # pylint: disable=protected-access + else: + self._def_entity(QAPISchemaObjectType(name, info, doc, ifcond, + None, members, None, [])) + return name + + def _def_enum_type(self, expr, info, doc): + name = expr['enum'] + data = expr['data'] + prefix = expr.get('prefix') + ifcond = expr.get('if') + self._def_entity(QAPISchemaEnumType( + name, info, doc, ifcond, + self._make_enum_members(data, info), prefix)) + + def _make_member(self, name, typ, ifcond, info): + optional = False + if name.startswith('*'): + name = name[1:] + optional = True + if isinstance(typ, list): + assert len(typ) == 1 + typ = self._make_array_type(typ[0], info) + return QAPISchemaObjectTypeMember(name, info, typ, optional, ifcond) + + def _make_members(self, data, info): + return [self._make_member(key, value['type'], value.get('if'), info) + for (key, value) in data.items()] + + def _def_struct_type(self, expr, info, doc): + name = expr['struct'] + base = expr.get('base') + data = expr['data'] + ifcond = expr.get('if') + features = expr.get('features', []) + self._def_entity(QAPISchemaObjectType( + name, info, doc, ifcond, base, + self._make_members(data, info), + None, + self._make_features(features, info))) + + def _make_variant(self, case, typ, ifcond, info): + return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) + + def _make_simple_variant(self, case, typ, ifcond, info): + if isinstance(typ, list): + assert len(typ) == 1 + typ = self._make_array_type(typ[0], info) + typ = self._make_implicit_object_type( + typ, info, None, self.lookup_type(typ), + 'wrapper', [self._make_member('data', typ, None, info)]) + return QAPISchemaObjectTypeVariant(case, info, typ, ifcond) + + def _def_union_type(self, expr, info, doc): + name = expr['union'] + data = expr['data'] + base = expr.get('base') + ifcond = expr.get('if') + tag_name = expr.get('discriminator') + tag_member = None + if isinstance(base, dict): + base = self._make_implicit_object_type( + name, info, doc, ifcond, + 'base', self._make_members(base, info)) + if tag_name: + variants = [self._make_variant(key, value['type'], + value.get('if'), info) + for (key, value) in data.items()] + members = [] + else: + variants = [self._make_simple_variant(key, value['type'], + value.get('if'), info) + for (key, value) in data.items()] + enum = [{'name': v.name, 'if': v.ifcond} for v in variants] + typ = self._make_implicit_enum_type(name, info, ifcond, enum) + tag_member = QAPISchemaObjectTypeMember('type', info, typ, False) + members = [tag_member] + self._def_entity( + QAPISchemaObjectType(name, info, doc, ifcond, base, members, + QAPISchemaObjectTypeVariants( + tag_name, info, tag_member, variants), + [])) + + def _def_alternate_type(self, expr, info, doc): + name = expr['alternate'] + data = expr['data'] + ifcond = expr.get('if') + variants = [self._make_variant(key, value['type'], value.get('if'), + info) + for (key, value) in data.items()] + tag_member = QAPISchemaObjectTypeMember('type', info, 'QType', False) + self._def_entity( + QAPISchemaAlternateType(name, info, doc, ifcond, + QAPISchemaObjectTypeVariants( + None, info, tag_member, variants))) + + def _def_command(self, expr, info, doc): + name = expr['command'] + data = expr.get('data') + rets = expr.get('returns') + gen = expr.get('gen', True) + success_response = expr.get('success-response', True) + boxed = expr.get('boxed', False) + allow_oob = expr.get('allow-oob', False) + allow_preconfig = expr.get('allow-preconfig', False) + ifcond = expr.get('if') + if isinstance(data, OrderedDict): + data = self._make_implicit_object_type( + name, info, doc, ifcond, 'arg', self._make_members(data, info)) + if isinstance(rets, list): + assert len(rets) == 1 + rets = self._make_array_type(rets[0], info) + self._def_entity(QAPISchemaCommand(name, info, doc, ifcond, data, rets, + gen, success_response, + boxed, allow_oob, allow_preconfig)) + + def _def_event(self, expr, info, doc): + name = expr['event'] + data = expr.get('data') + boxed = expr.get('boxed', False) + ifcond = expr.get('if') + if isinstance(data, OrderedDict): + data = self._make_implicit_object_type( + name, info, doc, ifcond, 'arg', self._make_members(data, info)) + self._def_entity(QAPISchemaEvent(name, info, doc, ifcond, data, boxed)) + + def _def_exprs(self, exprs): + for expr_elem in exprs: + expr = expr_elem['expr'] + info = expr_elem['info'] + doc = expr_elem.get('doc') + if 'enum' in expr: + self._def_enum_type(expr, info, doc) + elif 'struct' in expr: + self._def_struct_type(expr, info, doc) + elif 'union' in expr: + self._def_union_type(expr, info, doc) + elif 'alternate' in expr: + self._def_alternate_type(expr, info, doc) + elif 'command' in expr: + self._def_command(expr, info, doc) + elif 'event' in expr: + self._def_event(expr, info, doc) + elif 'include' in expr: + self._def_include(expr, info, doc) + else: + assert False + + def check(self): + for ent in self._entity_list: + ent.check(self) + + def visit(self, visitor): + visitor.visit_begin(self) + module = None + visitor.visit_module(module) + for entity in self._entity_list: + if visitor.visit_needed(entity): + if entity.module != module: + module = entity.module + visitor.visit_module(module) + entity.visit(visitor) + visitor.visit_end() diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py new file mode 100644 index 0000000000..8956885033 --- /dev/null +++ b/scripts/qapi/source.py @@ -0,0 +1,67 @@ +# +# QAPI frontend source file info +# +# Copyright (c) 2019 Red Hat Inc. +# +# Authors: +# Markus Armbruster <armbru@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +import copy +import sys + + +class QAPISchemaPragma(object): + def __init__(self): + # Are documentation comments required? + self.doc_required = False + # Whitelist of commands allowed to return a non-dictionary + self.returns_whitelist = [] + # Whitelist of entities allowed to violate case conventions + self.name_case_whitelist = [] + + +class QAPISourceInfo(object): + def __init__(self, fname, line, parent): + self.fname = fname + self.line = line + self.parent = parent + self.pragma = parent.pragma if parent else QAPISchemaPragma() + self.defn_meta = None + self.defn_name = None + + def set_defn(self, meta, name): + self.defn_meta = meta + self.defn_name = name + + def next_line(self): + info = copy.copy(self) + info.line += 1 + return info + + def loc(self): + if self.fname is None: + return sys.argv[0] + ret = self.fname + if self.line is not None: + ret += ':%d' % self.line + return ret + + def in_defn(self): + if self.defn_name: + return "%s: In %s '%s':\n" % (self.fname, + self.defn_meta, self.defn_name) + return '' + + def include_path(self): + ret = '' + parent = self.parent + while parent: + ret = 'In file included from %s:\n' % parent.loc() + ret + parent = parent.parent + return ret + + def __str__(self): + return self.include_path() + self.in_defn() + self.loc() diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py index 711543147d..d8751daa04 100644 --- a/scripts/qapi/types.py +++ b/scripts/qapi/types.py @@ -14,6 +14,8 @@ This work is licensed under the terms of the GNU GPL, version 2. """ from qapi.common import * +from qapi.gen import QAPISchemaModularCVisitor, ifcontext +from qapi.schema import QAPISchemaEnumMember, QAPISchemaObjectType # variants must be emitted before their container; track what has already diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py index 484ebb66ad..c72f2bc5c0 100644 --- a/scripts/qapi/visit.py +++ b/scripts/qapi/visit.py @@ -14,6 +14,8 @@ See the COPYING file in the top-level directory. """ from qapi.common import * +from qapi.gen import QAPISchemaModularCVisitor, ifcontext +from qapi.schema import QAPISchemaObjectType def gen_visit_decl(name, scalar=False): diff --git a/tests/Makefile.include b/tests/Makefile.include index 1b24b8ba10..09e5b410dc 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -31,13 +31,20 @@ ifneq ($(wildcard config-host.mak),) export SRC_PATH # TODO don't duplicate $(SRC_PATH)/Makefile's qapi-py here -qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \ +qapi-py = $(SRC_PATH)/scripts/qapi/__init__.py \ +$(SRC_PATH)/scripts/qapi/commands.py \ +$(SRC_PATH)/scripts/qapi/common.py \ +$(SRC_PATH)/scripts/qapi/doc.py \ +$(SRC_PATH)/scripts/qapi/error.py \ $(SRC_PATH)/scripts/qapi/events.py \ +$(SRC_PATH)/scripts/qapi/expr.py \ +$(SRC_PATH)/scripts/qapi/gen.py \ $(SRC_PATH)/scripts/qapi/introspect.py \ +$(SRC_PATH)/scripts/qapi/parser.py \ +$(SRC_PATH)/scripts/qapi/schema.py \ +$(SRC_PATH)/scripts/qapi/source.py \ $(SRC_PATH)/scripts/qapi/types.py \ $(SRC_PATH)/scripts/qapi/visit.py \ -$(SRC_PATH)/scripts/qapi/common.py \ -$(SRC_PATH)/scripts/qapi/doc.py \ $(SRC_PATH)/scripts/qapi-gen.py # Get the list of all supported sysemu targets diff --git a/tests/qapi-schema/test-qapi.py b/tests/qapi-schema/test-qapi.py index 42baa702b6..664254618a 100755 --- a/tests/qapi-schema/test-qapi.py +++ b/tests/qapi-schema/test-qapi.py @@ -12,15 +12,19 @@ # from __future__ import print_function + import argparse import difflib import os import sys + +from qapi.error import QAPIError +from qapi.schema import QAPISchema, QAPISchemaVisitor + if sys.version_info[0] < 3: from cStringIO import StringIO else: from io import StringIO -from qapi.common import QAPIError, QAPISchema, QAPISchemaVisitor class QAPISchemaTestVisitor(QAPISchemaVisitor): |