qapi: Split up scripts/qapi/common.py

The QAPI code generator clocks in at some 3100 SLOC in 8 source files.
Almost 60% of the code is in qapi/common.py.  Split it into more
focused modules:

* Move QAPISchemaPragma and QAPISourceInfo to qapi/source.py.

* Move QAPIError and its sub-classes to qapi/error.py.

* Move QAPISchemaParser and QAPIDoc to parser.py.  Use the opportunity
  to put QAPISchemaParser first.

* Move check_expr() & friends to qapi/expr.py.  Use the opportunity to
  put the code into a more sensible order.

* Move QAPISchema & friends to qapi/schema.py

* Move QAPIGen and its sub-classes, ifcontext,
  QAPISchemaModularCVisitor, and QAPISchemaModularCVisitor to qapi/gen.py

* Delete camel_case(), it's unused since commit e98859a9b9 "qapi:
  Clean up after recent conversions to QAPISchemaVisitor"

A number of helper functions remain in qapi/common.py.  I considered
moving the code generator helpers to qapi/gen.py, but decided not to.
Perhaps we should rewrite them as methods of QAPIGen some day.

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20191018074345.24034-7-armbru@redhat.com>
[Add "# -*- coding: utf-8 -*-" lines]

1 files changed, 570 insertions, 0 deletions

diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py
new file mode 100644
index 0000000000..e800876ad1
--- /dev/null
+++ b/scripts/qapi/parser.py
@@ -0,0 +1,570 @@
+# -*- coding: utf-8 -*-
+#
+# QAPI schema parser
+#
+# Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2019 Red Hat Inc.
+#
+# Authors:
+#  Anthony Liguori <aliguori@us.ibm.com>
+#  Markus Armbruster <armbru@redhat.com>
+#  Marc-André Lureau <marcandre.lureau@redhat.com>
+#  Kevin Wolf <kwolf@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+import os
+import re
+import sys
+from collections import OrderedDict
+
+from qapi.error import QAPIParseError, QAPISemError
+from qapi.source import QAPISourceInfo
+
+
+class QAPISchemaParser(object):
+
+    def __init__(self, fname, previously_included=None, incl_info=None):
+        previously_included = previously_included or set()
+        previously_included.add(os.path.abspath(fname))
+
+        try:
+            if sys.version_info[0] >= 3:
+                fp = open(fname, 'r', encoding='utf-8')
+            else:
+                fp = open(fname, 'r')
+            self.src = fp.read()
+        except IOError as e:
+            raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
+                               "can't read %s file '%s': %s"
+                               % ("include" if incl_info else "schema",
+                                  fname,
+                                  e.strerror))
+
+        if self.src == '' or self.src[-1] != '\n':
+            self.src += '\n'
+        self.cursor = 0
+        self.info = QAPISourceInfo(fname, 1, incl_info)
+        self.line_pos = 0
+        self.exprs = []
+        self.docs = []
+        self.accept()
+        cur_doc = None
+
+        while self.tok is not None:
+            info = self.info
+            if self.tok == '#':
+                self.reject_expr_doc(cur_doc)
+                cur_doc = self.get_doc(info)
+                self.docs.append(cur_doc)
+                continue
+
+            expr = self.get_expr(False)
+            if 'include' in expr:
+                self.reject_expr_doc(cur_doc)
+                if len(expr) != 1:
+                    raise QAPISemError(info, "invalid 'include' directive")
+                include = expr['include']
+                if not isinstance(include, str):
+                    raise QAPISemError(info,
+                                       "value of 'include' must be a string")
+                incl_fname = os.path.join(os.path.dirname(fname),
+                                          include)
+                self.exprs.append({'expr': {'include': incl_fname},
+                                   'info': info})
+                exprs_include = self._include(include, info, incl_fname,
+                                              previously_included)
+                if exprs_include:
+                    self.exprs.extend(exprs_include.exprs)
+                    self.docs.extend(exprs_include.docs)
+            elif "pragma" in expr:
+                self.reject_expr_doc(cur_doc)
+                if len(expr) != 1:
+                    raise QAPISemError(info, "invalid 'pragma' directive")
+                pragma = expr['pragma']
+                if not isinstance(pragma, dict):
+                    raise QAPISemError(
+                        info, "value of 'pragma' must be an object")
+                for name, value in pragma.items():
+                    self._pragma(name, value, info)
+            else:
+                expr_elem = {'expr': expr,
+                             'info': info}
+                if cur_doc:
+                    if not cur_doc.symbol:
+                        raise QAPISemError(
+                            cur_doc.info, "definition documentation required")
+                    expr_elem['doc'] = cur_doc
+                self.exprs.append(expr_elem)
+            cur_doc = None
+        self.reject_expr_doc(cur_doc)
+
+    @staticmethod
+    def reject_expr_doc(doc):
+        if doc and doc.symbol:
+            raise QAPISemError(
+                doc.info,
+                "documentation for '%s' is not followed by the definition"
+                % doc.symbol)
+
+    def _include(self, include, info, incl_fname, previously_included):
+        incl_abs_fname = os.path.abspath(incl_fname)
+        # catch inclusion cycle
+        inf = info
+        while inf:
+            if incl_abs_fname == os.path.abspath(inf.fname):
+                raise QAPISemError(info, "inclusion loop for %s" % include)
+            inf = inf.parent
+
+        # skip multiple include of the same file
+        if incl_abs_fname in previously_included:
+            return None
+
+        return QAPISchemaParser(incl_fname, previously_included, info)
+
+    def _pragma(self, name, value, info):
+        if name == 'doc-required':
+            if not isinstance(value, bool):
+                raise QAPISemError(info,
+                                   "pragma 'doc-required' must be boolean")
+            info.pragma.doc_required = value
+        elif name == 'returns-whitelist':
+            if (not isinstance(value, list)
+                    or any([not isinstance(elt, str) for elt in value])):
+                raise QAPISemError(
+                    info,
+                    "pragma returns-whitelist must be a list of strings")
+            info.pragma.returns_whitelist = value
+        elif name == 'name-case-whitelist':
+            if (not isinstance(value, list)
+                    or any([not isinstance(elt, str) for elt in value])):
+                raise QAPISemError(
+                    info,
+                    "pragma name-case-whitelist must be a list of strings")
+            info.pragma.name_case_whitelist = value
+        else:
+            raise QAPISemError(info, "unknown pragma '%s'" % name)
+
+    def accept(self, skip_comment=True):
+        while True:
+            self.tok = self.src[self.cursor]
+            self.pos = self.cursor
+            self.cursor += 1
+            self.val = None
+
+            if self.tok == '#':
+                if self.src[self.cursor] == '#':
+                    # Start of doc comment
+                    skip_comment = False
+                self.cursor = self.src.find('\n', self.cursor)
+                if not skip_comment:
+                    self.val = self.src[self.pos:self.cursor]
+                    return
+            elif self.tok in '{}:,[]':
+                return
+            elif self.tok == "'":
+                # Note: we accept only printable ASCII
+                string = ''
+                esc = False
+                while True:
+                    ch = self.src[self.cursor]
+                    self.cursor += 1
+                    if ch == '\n':
+                        raise QAPIParseError(self, "missing terminating \"'\"")
+                    if esc:
+                        # Note: we recognize only \\ because we have
+                        # no use for funny characters in strings
+                        if ch != '\\':
+                            raise QAPIParseError(self,
+                                                 "unknown escape \\%s" % ch)
+                        esc = False
+                    elif ch == '\\':
+                        esc = True
+                        continue
+                    elif ch == "'":
+                        self.val = string
+                        return
+                    if ord(ch) < 32 or ord(ch) >= 127:
+                        raise QAPIParseError(
+                            self, "funny character in string")
+                    string += ch
+            elif self.src.startswith('true', self.pos):
+                self.val = True
+                self.cursor += 3
+                return
+            elif self.src.startswith('false', self.pos):
+                self.val = False
+                self.cursor += 4
+                return
+            elif self.tok == '\n':
+                if self.cursor == len(self.src):
+                    self.tok = None
+                    return
+                self.info = self.info.next_line()
+                self.line_pos = self.cursor
+            elif not self.tok.isspace():
+                # Show up to next structural, whitespace or quote
+                # character
+                match = re.match('[^[\\]{}:,\\s\'"]+',
+                                 self.src[self.cursor-1:])
+                raise QAPIParseError(self, "stray '%s'" % match.group(0))
+
+    def get_members(self):
+        expr = OrderedDict()
+        if self.tok == '}':
+            self.accept()
+            return expr
+        if self.tok != "'":
+            raise QAPIParseError(self, "expected string or '}'")
+        while True:
+            key = self.val
+            self.accept()
+            if self.tok != ':':
+                raise QAPIParseError(self, "expected ':'")
+            self.accept()
+            if key in expr:
+                raise QAPIParseError(self, "duplicate key '%s'" % key)
+            expr[key] = self.get_expr(True)
+            if self.tok == '}':
+                self.accept()
+                return expr
+            if self.tok != ',':
+                raise QAPIParseError(self, "expected ',' or '}'")
+            self.accept()
+            if self.tok != "'":
+                raise QAPIParseError(self, "expected string")
+
+    def get_values(self):
+        expr = []
+        if self.tok == ']':
+            self.accept()
+            return expr
+        if self.tok not in "{['tfn":
+            raise QAPIParseError(
+                self, "expected '{', '[', ']', string, boolean or 'null'")
+        while True:
+            expr.append(self.get_expr(True))
+            if self.tok == ']':
+                self.accept()
+                return expr
+            if self.tok != ',':
+                raise QAPIParseError(self, "expected ',' or ']'")
+            self.accept()
+
+    def get_expr(self, nested):
+        if self.tok != '{' and not nested:
+            raise QAPIParseError(self, "expected '{'")
+        if self.tok == '{':
+            self.accept()
+            expr = self.get_members()
+        elif self.tok == '[':
+            self.accept()
+            expr = self.get_values()
+        elif self.tok in "'tfn":
+            expr = self.val
+            self.accept()
+        else:
+            raise QAPIParseError(
+                self, "expected '{', '[', string, boolean or 'null'")
+        return expr
+
+    def get_doc(self, info):
+        if self.val != '##':
+            raise QAPIParseError(
+                self, "junk after '##' at start of documentation comment")
+
+        doc = QAPIDoc(self, info)
+        self.accept(False)
+        while self.tok == '#':
+            if self.val.startswith('##'):
+                # End of doc comment
+                if self.val != '##':
+                    raise QAPIParseError(
+                        self,
+                        "junk after '##' at end of documentation comment")
+                doc.end_comment()
+                self.accept()
+                return doc
+            else:
+                doc.append(self.val)
+            self.accept(False)
+
+        raise QAPIParseError(self, "documentation comment must end with '##'")
+
+
+class QAPIDoc(object):
+    """
+    A documentation comment block, either definition or free-form
+
+    Definition documentation blocks consist of
+
+    * a body section: one line naming the definition, followed by an
+      overview (any number of lines)
+
+    * argument sections: a description of each argument (for commands
+      and events) or member (for structs, unions and alternates)
+
+    * features sections: a description of each feature flag
+
+    * additional (non-argument) sections, possibly tagged
+
+    Free-form documentation blocks consist only of a body section.
+    """
+
+    class Section(object):
+        def __init__(self, name=None):
+            # optional section name (argument/member or section name)
+            self.name = name
+            # the list of lines for this section
+            self.text = ''
+
+        def append(self, line):
+            self.text += line.rstrip() + '\n'
+
+    class ArgSection(Section):
+        def __init__(self, name):
+            QAPIDoc.Section.__init__(self, name)
+            self.member = None
+
+        def connect(self, member):
+            self.member = member
+
+    def __init__(self, parser, info):
+        # self._parser is used to report errors with QAPIParseError.  The
+        # resulting error position depends on the state of the parser.
+        # It happens to be the beginning of the comment.  More or less
+        # servicable, but action at a distance.
+        self._parser = parser
+        self.info = info
+        self.symbol = None
+        self.body = QAPIDoc.Section()
+        # dict mapping parameter name to ArgSection
+        self.args = OrderedDict()
+        self.features = OrderedDict()
+        # a list of Section
+        self.sections = []
+        # the current section
+        self._section = self.body
+        self._append_line = self._append_body_line
+
+    def has_section(self, name):
+        """Return True if we have a section with this name."""
+        for i in self.sections:
+            if i.name == name:
+                return True
+        return False
+
+    def append(self, line):
+        """
+        Parse a comment line and add it to the documentation.
+
+        The way that the line is dealt with depends on which part of
+        the documentation we're parsing right now:
+        * The body section: ._append_line is ._append_body_line
+        * An argument section: ._append_line is ._append_args_line
+        * A features section: ._append_line is ._append_features_line
+        * An additional section: ._append_line is ._append_various_line
+        """
+        line = line[1:]
+        if not line:
+            self._append_freeform(line)
+            return
+
+        if line[0] != ' ':
+            raise QAPIParseError(self._parser, "missing space after #")
+        line = line[1:]
+        self._append_line(line)
+
+    def end_comment(self):
+        self._end_section()
+
+    @staticmethod
+    def _is_section_tag(name):
+        return name in ('Returns:', 'Since:',
+                        # those are often singular or plural
+                        'Note:', 'Notes:',
+                        'Example:', 'Examples:',
+                        'TODO:')
+
+    def _append_body_line(self, line):
+        """
+        Process a line of documentation text in the body section.
+
+        If this a symbol line and it is the section's first line, this
+        is a definition documentation block for that symbol.
+
+        If it's a definition documentation block, another symbol line
+        begins the argument section for the argument named by it, and
+        a section tag begins an additional section.  Start that
+        section and append the line to it.
+
+        Else, append the line to the current section.
+        """
+        name = line.split(' ', 1)[0]
+        # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
+        # recognized, and get silently treated as ordinary text
+        if not self.symbol and not self.body.text and line.startswith('@'):
+            if not line.endswith(':'):
+                raise QAPIParseError(self._parser, "line should end with ':'")
+            self.symbol = line[1:-1]
+            # FIXME invalid names other than the empty string aren't flagged
+            if not self.symbol:
+                raise QAPIParseError(self._parser, "invalid name")
+        elif self.symbol:
+            # This is a definition documentation block
+            if name.startswith('@') and name.endswith(':'):
+                self._append_line = self._append_args_line
+                self._append_args_line(line)
+            elif line == 'Features:':
+                self._append_line = self._append_features_line
+            elif self._is_section_tag(name):
+                self._append_line = self._append_various_line
+                self._append_various_line(line)
+            else:
+                self._append_freeform(line.strip())
+        else:
+            # This is a free-form documentation block
+            self._append_freeform(line.strip())
+
+    def _append_args_line(self, line):
+        """
+        Process a line of documentation text in an argument section.
+
+        A symbol line begins the next argument section, a section tag
+        section or a non-indented line after a blank line begins an
+        additional section.  Start that section and append the line to
+        it.
+
+        Else, append the line to the current section.
+
+        """
+        name = line.split(' ', 1)[0]
+
+        if name.startswith('@') and name.endswith(':'):
+            line = line[len(name)+1:]
+            self._start_args_section(name[1:-1])
+        elif self._is_section_tag(name):
+            self._append_line = self._append_various_line
+            self._append_various_line(line)
+            return
+        elif (self._section.text.endswith('\n\n')
+              and line and not line[0].isspace()):
+            if line == 'Features:':
+                self._append_line = self._append_features_line
+            else:
+                self._start_section()
+                self._append_line = self._append_various_line
+                self._append_various_line(line)
+            return
+
+        self._append_freeform(line.strip())
+
+    def _append_features_line(self, line):
+        name = line.split(' ', 1)[0]
+
+        if name.startswith('@') and name.endswith(':'):
+            line = line[len(name)+1:]
+            self._start_features_section(name[1:-1])
+        elif self._is_section_tag(name):
+            self._append_line = self._append_various_line
+            self._append_various_line(line)
+            return
+        elif (self._section.text.endswith('\n\n')
+              and line and not line[0].isspace()):
+            self._start_section()
+            self._append_line = self._append_various_line
+            self._append_various_line(line)
+            return
+
+        self._append_freeform(line.strip())
+
+    def _append_various_line(self, line):
+        """
+        Process a line of documentation text in an additional section.
+
+        A symbol line is an error.
+
+        A section tag begins an additional section.  Start that
+        section and append the line to it.
+
+        Else, append the line to the current section.
+        """
+        name = line.split(' ', 1)[0]
+
+        if name.startswith('@') and name.endswith(':'):
+            raise QAPIParseError(self._parser,
+                                 "'%s' can't follow '%s' section"
+                                 % (name, self.sections[0].name))
+        elif self._is_section_tag(name):
+            line = line[len(name)+1:]
+            self._start_section(name[:-1])
+
+        if (not self._section.name or
+                not self._section.name.startswith('Example')):
+            line = line.strip()
+
+        self._append_freeform(line)
+
+    def _start_symbol_section(self, symbols_dict, name):
+        # FIXME invalid names other than the empty string aren't flagged
+        if not name:
+            raise QAPIParseError(self._parser, "invalid parameter name")
+        if name in symbols_dict:
+            raise QAPIParseError(self._parser,
+                                 "'%s' parameter name duplicated" % name)
+        assert not self.sections
+        self._end_section()
+        self._section = QAPIDoc.ArgSection(name)
+        symbols_dict[name] = self._section
+
+    def _start_args_section(self, name):
+        self._start_symbol_section(self.args, name)
+
+    def _start_features_section(self, name):
+        self._start_symbol_section(self.features, name)
+
+    def _start_section(self, name=None):
+        if name in ('Returns', 'Since') and self.has_section(name):
+            raise QAPIParseError(self._parser,
+                                 "duplicated '%s' section" % name)
+        self._end_section()
+        self._section = QAPIDoc.Section(name)
+        self.sections.append(self._section)
+
+    def _end_section(self):
+        if self._section:
+            text = self._section.text = self._section.text.strip()
+            if self._section.name and (not text or text.isspace()):
+                raise QAPIParseError(
+                    self._parser,
+                    "empty doc section '%s'" % self._section.name)
+            self._section = None
+
+    def _append_freeform(self, line):
+        match = re.match(r'(@\S+:)', line)
+        if match:
+            raise QAPIParseError(self._parser,
+                                 "'%s' not allowed in free-form documentation"
+                                 % match.group(1))
+        self._section.append(line)
+
+    def connect_member(self, member):
+        if member.name not in self.args:
+            # Undocumented TODO outlaw
+            self.args[member.name] = QAPIDoc.ArgSection(member.name)
+        self.args[member.name].connect(member)
+
+    def check_expr(self, expr):
+        if self.has_section('Returns') and 'command' not in expr:
+            raise QAPISemError(self.info,
+                               "'Returns:' is only valid for commands")
+
+    def check(self):
+        bogus = [name for name, section in self.args.items()
+                 if not section.member]
+        if bogus:
+            raise QAPISemError(
+                self.info,
+                "the following documented members are not in "
+                "the declaration: %s" % ", ".join(bogus))