From 7c1e1d5481fe8d2e757469179f9ccd14e8838ed1 Mon Sep 17 00:00:00 2001
From: Marc-André Lureau
Date: Thu, 23 Aug 2018 18:39:58 +0200
Subject: json: remove useless return value from lexer/parser

The lexer always returns 0 when char feeding. Furthermore, none of the
caller care about the return value.

Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
Message-Id: <20180326150916.9602-10-marcandre.lureau@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Signed-off-by: Markus Armbruster <armbru@redhat.com>
Message-Id: <20180823164025.12553-32-armbru@redhat.com>
---
 include/qapi/qmp/json-streamer.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index 00d8a23af8..cb808cf27d 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -36,10 +36,10 @@ typedef struct JSONMessageParser
 void json_message_parser_init(JSONMessageParser *parser,
                               void (*func)(JSONMessageParser *, GQueue *));
 
-int json_message_parser_feed(JSONMessageParser *parser,
+void json_message_parser_feed(JSONMessageParser *parser,
                              const char *buffer, size_t size);
 
-int json_message_parser_flush(JSONMessageParser *parser);
+void json_message_parser_flush(JSONMessageParser *parser);
 
 void json_message_parser_destroy(JSONMessageParser *parser);
 
-- 
cgit v1.2.3-55-g7522


From 037f2440888a22bd00ea0d8e37a1b7ed7d2bba88 Mon Sep 17 00:00:00 2001
From: Markus Armbruster
Date: Thu, 23 Aug 2018 18:40:00 +0200
Subject: json: Have lexer call streamer directly

json_lexer_init() takes the function to process a token as an
argument.  It's always json_message_process_token().  Makes the code
harder to understand for no actual gain.  Drop the indirection.

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180823164025.12553-34-armbru@redhat.com>
---
 include/qapi/qmp/json-lexer.h    | 13 +++----------
 include/qapi/qmp/json-streamer.h |  3 +++
 qobject/json-lexer.c             | 13 ++++++++-----
 qobject/json-streamer.c          |  6 +++---
 4 files changed, 17 insertions(+), 18 deletions(-)

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-lexer.h b/include/qapi/qmp/json-lexer.h
index 66ccf0357c..44bcf2ca64 100644
--- a/include/qapi/qmp/json-lexer.h
+++ b/include/qapi/qmp/json-lexer.h
@@ -32,20 +32,13 @@ typedef enum json_token_type {
     JSON_ERROR,
 } JSONTokenType;
 
-typedef struct JSONLexer JSONLexer;
-
-typedef void (JSONLexerEmitter)(JSONLexer *, GString *,
-                                JSONTokenType, int x, int y);
-
-struct JSONLexer
-{
-    JSONLexerEmitter *emit;
+typedef struct JSONLexer {
     int state;
     GString *token;
     int x, y;
-};
+} JSONLexer;
 
-void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func);
+void json_lexer_init(JSONLexer *lexer);
 
 void json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size);
 
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index cb808cf27d..7922e185a5 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -33,6 +33,9 @@ typedef struct JSONMessageParser
     uint64_t token_size;
 } JSONMessageParser;
 
+void json_message_process_token(JSONLexer *lexer, GString *input,
+                                JSONTokenType type, int x, int y);
+
 void json_message_parser_init(JSONMessageParser *parser,
                               void (*func)(JSONMessageParser *, GQueue *));
 
diff --git a/qobject/json-lexer.c b/qobject/json-lexer.c
index d9701f857b..17272a3874 100644
--- a/qobject/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -14,6 +14,7 @@
 #include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
@@ -278,9 +279,8 @@ static const uint8_t json_lexer[][256] =  {
     },
 };
 
-void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func)
+void json_lexer_init(JSONLexer *lexer)
 {
-    lexer->emit = func;
     lexer->state = IN_START;
     lexer->token = g_string_sized_new(3);
     lexer->x = lexer->y = 0;
@@ -316,7 +316,8 @@ static void json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
         case JSON_FLOAT:
         case JSON_KEYWORD:
         case JSON_STRING:
-            lexer->emit(lexer, lexer->token, new_state, lexer->x, lexer->y);
+            json_message_process_token(lexer, lexer->token, new_state,
+                                       lexer->x, lexer->y);
             /* fall through */
         case JSON_SKIP:
             g_string_truncate(lexer->token, 0);
@@ -336,7 +337,8 @@ static void json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
              * never a valid ASCII/UTF-8 sequence, so this should reliably
              * induce an error/flush state.
              */
-            lexer->emit(lexer, lexer->token, JSON_ERROR, lexer->x, lexer->y);
+            json_message_process_token(lexer, lexer->token, JSON_ERROR,
+                                       lexer->x, lexer->y);
             g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             lexer->state = new_state;
@@ -351,7 +353,8 @@ static void json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
      * this is a security consideration.
      */
     if (lexer->token->len > MAX_TOKEN_SIZE) {
-        lexer->emit(lexer, lexer->token, lexer->state, lexer->x, lexer->y);
+        json_message_process_token(lexer, lexer->token, lexer->state,
+                                   lexer->x, lexer->y);
         g_string_truncate(lexer->token, 0);
         lexer->state = IN_START;
     }
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 78dfff2aa0..9f57ebf2bd 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -34,8 +34,8 @@ static void json_message_free_tokens(JSONMessageParser *parser)
     }
 }
 
-static void json_message_process_token(JSONLexer *lexer, GString *input,
-                                       JSONTokenType type, int x, int y)
+void json_message_process_token(JSONLexer *lexer, GString *input,
+                                JSONTokenType type, int x, int y)
 {
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
     JSONToken *token;
@@ -115,7 +115,7 @@ void json_message_parser_init(JSONMessageParser *parser,
     parser->tokens = g_queue_new();
     parser->token_size = 0;
 
-    json_lexer_init(&parser->lexer, json_message_process_token);
+    json_lexer_init(&parser->lexer);
 }
 
 void json_message_parser_feed(JSONMessageParser *parser,
-- 
cgit v1.2.3-55-g7522


From 62815d85aed71eff7b6c3a524705180fb04f5d30 Mon Sep 17 00:00:00 2001
From: Markus Armbruster
Date: Thu, 23 Aug 2018 18:40:01 +0200
Subject: json: Redesign the callback to consume JSON values

The classical way to structure parser and lexer is to have the client
call the parser to get an abstract syntax tree, the parser call the
lexer to get the next token, and the lexer call some function to get
input characters.

Another way to structure them would be to have the client feed
characters to the lexer, the lexer feed tokens to the parser, and the
parser feed abstract syntax trees to some callback provided by the
client.  This way is more easily integrated into an event loop that
dispatches input characters as they arrive.

Our JSON parser is kind of between the two.  The lexer feeds tokens to
a "streamer" instead of a real parser.  The streamer accumulates
tokens until it got the sequence of tokens that comprise a single JSON
value (it counts curly braces and square brackets to decide).  It
feeds those token sequences to a callback provided by the client.  The
callback passes each token sequence to the parser, and gets back an
abstract syntax tree.

I figure it was done that way to make a straightforward recursive
descent parser possible.  "Get next token" becomes "pop the first
token off the token sequence".  Drawback: we need to store a complete
token sequence.  Each token eats 13 + input characters + malloc
overhead bytes.

Observations:

1. This is not the only way to use recursive descent.  If we replaced
   "get next token" by a coroutine yield, we could do without a
   streamer.

2. The lexer reports errors by passing a JSON_ERROR token to the
   streamer.  This communicates the offending input characters and
   their location, but no more.

3. The streamer reports errors by passing a null token sequence to the
   callback.  The (already poor) lexical error information is thrown
   away.

4. Having the callback receive a token sequence duplicates the code to
   convert token sequence to abstract syntax tree in every callback.

5. Known bug: the streamer silently drops incomplete token sequences.

This commit rectifies 4. by lifting the call of the parser from the
callbacks into the streamer.  Later commits will address 3. and 5.

The lifting removes a bug from qjson.c's parse_json(): it passed a
pointer to a non-null Error * in certain cases, as demonstrated by
check-qjson.c.

json_parser_parse() is now unused.  It's a stupid wrapper around
json_parser_parse_err().  Drop it, and rename json_parser_parse_err()
to json_parser_parse().

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180823164025.12553-35-armbru@redhat.com>
---
 include/qapi/qmp/json-parser.h   |  3 +--
 include/qapi/qmp/json-streamer.h |  8 ++++++--
 monitor.c                        | 18 ++++++++----------
 qapi/qmp-dispatch.c              |  1 -
 qga/main.c                       | 12 +++---------
 qobject/json-parser.c            |  7 +------
 qobject/json-streamer.c          | 19 +++++++++++--------
 qobject/qjson.c                  | 14 +++++---------
 tests/check-qjson.c              |  1 -
 tests/libqtest.c                 | 10 ++++------
 10 files changed, 39 insertions(+), 54 deletions(-)

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h
index 102f5c0068..a34209db7a 100644
--- a/include/qapi/qmp/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -16,7 +16,6 @@
 
 #include "qemu-common.h"
 
-QObject *json_parser_parse(GQueue *tokens, va_list *ap);
-QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
+QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp);
 
 #endif
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index 7922e185a5..e162fd01da 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -25,7 +25,9 @@ typedef struct JSONToken {
 
 typedef struct JSONMessageParser
 {
-    void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
+    void (*emit)(void *opaque, QObject *json, Error *err);
+    void *opaque;
+    va_list *ap;
     JSONLexer lexer;
     int brace_count;
     int bracket_count;
@@ -37,7 +39,9 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
                                 JSONTokenType type, int x, int y);
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, GQueue *));
+                              void (*emit)(void *opaque, QObject *json,
+                                           Error *err),
+                              void *opaque, va_list *ap);
 
 void json_message_parser_feed(JSONMessageParser *parser,
                              const char *buffer, size_t size);
diff --git a/monitor.c b/monitor.c
index 94f673511b..08f799a7bb 100644
--- a/monitor.c
+++ b/monitor.c
@@ -59,7 +59,6 @@
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/json-streamer.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qlist.h"
 #include "qom/object_interfaces.h"
 #include "trace-root.h"
@@ -4256,18 +4255,15 @@ static void monitor_qmp_bh_dispatcher(void *data)
 
 #define  QMP_REQ_QUEUE_LEN_MAX  (8)
 
-static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
+static void handle_qmp_command(void *opaque, QObject *req, Error *err)
 {
-    QObject *req, *id = NULL;
+    Monitor *mon = opaque;
+    QObject *id = NULL;
     QDict *qdict;
-    MonitorQMP *mon_qmp = container_of(parser, MonitorQMP, parser);
-    Monitor *mon = container_of(mon_qmp, Monitor, qmp);
-    Error *err = NULL;
     QMPRequest *req_obj;
 
-    req = json_parser_parse_err(tokens, NULL, &err);
     if (!req && !err) {
-        /* json_parser_parse_err() sucks: can fail without setting @err */
+        /* json_parser_parse() sucks: can fail without setting @err */
         error_setg(&err, QERR_JSON_PARSING);
     }
 
@@ -4465,7 +4461,8 @@ static void monitor_qmp_event(void *opaque, int event)
         monitor_qmp_response_flush(mon);
         monitor_qmp_cleanup_queues(mon);
         json_message_parser_destroy(&mon->qmp.parser);
-        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command,
+                                 mon, NULL);
         mon_refcount--;
         monitor_fdsets_cleanup();
         break;
@@ -4683,7 +4680,8 @@ void monitor_init(Chardev *chr, int flags)
 
     if (monitor_is_qmp(mon)) {
         qemu_chr_fe_set_echo(&mon->chr, true);
-        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command,
+                                 mon, NULL);
         if (mon->use_io_thread) {
             /*
              * Make sure the old iowatch is gone.  It's possible when
diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c
index 6f2d466596..d8da1a62de 100644
--- a/qapi/qmp-dispatch.c
+++ b/qapi/qmp-dispatch.c
@@ -14,7 +14,6 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qapi/qmp/dispatch.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qbool.h"
diff --git a/qga/main.c b/qga/main.c
index 87372d40ef..2fc49d00d8 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -19,7 +19,6 @@
 #include <sys/wait.h>
 #endif
 #include "qapi/qmp/json-streamer.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qstring.h"
@@ -597,18 +596,13 @@ static void process_command(GAState *s, QDict *req)
 }
 
 /* handle requests/control events coming in over the channel */
-static void process_event(JSONMessageParser *parser, GQueue *tokens)
+static void process_event(void *opaque, QObject *obj, Error *err)
 {
-    GAState *s = container_of(parser, GAState, parser);
-    QObject *obj;
+    GAState *s = opaque;
     QDict *req, *rsp;
-    Error *err = NULL;
     int ret;
 
-    g_assert(s && parser);
-
     g_debug("process_event: called");
-    obj = json_parser_parse_err(tokens, NULL, &err);
     if (err) {
         goto err;
     }
@@ -1320,7 +1314,7 @@ static int run_agent(GAState *s, GAConfig *config, int socket_activation)
     s->command_state = ga_command_state_new();
     ga_command_state_init(s, s->command_state);
     ga_command_state_init_all(s->command_state);
-    json_message_parser_init(&s->parser, process_event);
+    json_message_parser_init(&s->parser, process_event, s, NULL);
 
 #ifndef _WIN32
     if (!register_signal_handlers()) {
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index 7bfa08200c..95fa348e21 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -541,12 +541,7 @@ static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
     }
 }
 
-QObject *json_parser_parse(GQueue *tokens, va_list *ap)
-{
-    return json_parser_parse_err(tokens, ap, NULL);
-}
-
-QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp)
+QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp)
 {
     JSONParserContext ctxt = { .buf = tokens };
     QObject *result;
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 9f57ebf2bd..7fd0ff8756 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -14,6 +14,7 @@
 #include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
@@ -38,8 +39,9 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
                                 JSONTokenType type, int x, int y)
 {
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
+    Error *err = NULL;
     JSONToken *token;
-    GQueue *tokens;
+    QObject *json;
 
     switch (type) {
     case JSON_LCURLY:
@@ -97,19 +99,20 @@ out_emit:
     /* send current list of tokens to parser and reset tokenizer */
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    /* parser->emit takes ownership of parser->tokens.  Remove our own
-     * reference to parser->tokens before handing it out to parser->emit.
-     */
-    tokens = parser->tokens;
+    json = json_parser_parse(parser->tokens, parser->ap, &err);
     parser->tokens = g_queue_new();
-    parser->emit(parser, tokens);
     parser->token_size = 0;
+    parser->emit(parser->opaque, json, err);
 }
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, GQueue *))
+                              void (*emit)(void *opaque, QObject *json,
+                                           Error *err),
+                              void *opaque, va_list *ap)
 {
-    parser->emit = func;
+    parser->emit = emit;
+    parser->opaque = opaque;
+    parser->ap = ap;
     parser->brace_count = 0;
     parser->bracket_count = 0;
     parser->tokens = g_queue_new();
diff --git a/qobject/qjson.c b/qobject/qjson.c
index ab4040f235..7395556069 100644
--- a/qobject/qjson.c
+++ b/qobject/qjson.c
@@ -13,8 +13,6 @@
 
 #include "qemu/osdep.h"
 #include "qapi/error.h"
-#include "qapi/qmp/json-lexer.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qbool.h"
@@ -27,16 +25,16 @@
 typedef struct JSONParsingState
 {
     JSONMessageParser parser;
-    va_list *ap;
     QObject *result;
     Error *err;
 } JSONParsingState;
 
-static void parse_json(JSONMessageParser *parser, GQueue *tokens)
+static void consume_json(void *opaque, QObject *json, Error *err)
 {
-    JSONParsingState *s = container_of(parser, JSONParsingState, parser);
+    JSONParsingState *s = opaque;
 
-    s->result = json_parser_parse_err(tokens, s->ap, &s->err);
+    s->result = json;
+    error_propagate(&s->err, err);
 }
 
 /*
@@ -54,9 +52,7 @@ static QObject *qobject_from_jsonv(const char *string, va_list *ap,
 {
     JSONParsingState state = {};
 
-    state.ap = ap;
-
-    json_message_parser_init(&state.parser, parse_json);
+    json_message_parser_init(&state.parser, consume_json, &state, ap);
     json_message_parser_feed(&state.parser, string, strlen(string));
     json_message_parser_flush(&state.parser);
     json_message_parser_destroy(&state.parser);
diff --git a/tests/check-qjson.c b/tests/check-qjson.c
index defc21fa04..604886a1a2 100644
--- a/tests/check-qjson.c
+++ b/tests/check-qjson.c
@@ -1438,7 +1438,6 @@ static void multiple_values(void)
     qobject_unref(obj);
 
     /* BUG simultaneously succeeds and fails */
-    /* BUG calls json_parser_parse() with errp pointing to non-null */
     obj = qobject_from_json("} true", &err);
     g_assert(qbool_get_bool(qobject_to(QBool, obj)));
     error_free_or_abort(&err);
diff --git a/tests/libqtest.c b/tests/libqtest.c
index af2a24e796..1f3b0cb1b1 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -21,9 +21,9 @@
 #include <sys/un.h>
 
 #include "libqtest.h"
+#include "qemu-common.h"
 #include "qemu/cutils.h"
 #include "qapi/error.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
@@ -446,12 +446,10 @@ typedef struct {
     QDict *response;
 } QMPResponseParser;
 
-static void qmp_response(JSONMessageParser *parser, GQueue *tokens)
+static void qmp_response(void *opaque, QObject *obj, Error *err)
 {
-    QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser);
-    QObject *obj;
+    QMPResponseParser *qmp = opaque;
 
-    obj = json_parser_parse(tokens, NULL);
     if (!obj) {
         fprintf(stderr, "QMP JSON response parsing failed\n");
         abort();
@@ -468,7 +466,7 @@ QDict *qmp_fd_receive(int fd)
     bool log = getenv("QTEST_LOG") != NULL;
 
     qmp.response = NULL;
-    json_message_parser_init(&qmp.parser, qmp_response);
+    json_message_parser_init(&qmp.parser, qmp_response, &qmp, NULL);
     while (!qmp.response) {
         ssize_t len;
         char c;
-- 
cgit v1.2.3-55-g7522


From a2731e08ee8633fcdc2af944b8f8f315678f7669 Mon Sep 17 00:00:00 2001
From: Markus Armbruster
Date: Thu, 23 Aug 2018 18:40:17 +0200
Subject: json: Unbox tokens queue in JSONMessageParser

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180823164025.12553-51-armbru@redhat.com>
---
 include/qapi/qmp/json-streamer.h |  2 +-
 qobject/json-parser.c            |  1 -
 qobject/json-streamer.c          | 30 +++++++++++-------------------
 3 files changed, 12 insertions(+), 21 deletions(-)

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index e162fd01da..d1d7fe2595 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -31,7 +31,7 @@ typedef struct JSONMessageParser
     JSONLexer lexer;
     int brace_count;
     int bracket_count;
-    GQueue *tokens;
+    GQueue tokens;
     uint64_t token_size;
 } JSONMessageParser;
 
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index 685e9dac24..e9a9f937f3 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -552,7 +552,6 @@ QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp)
         parser_context_pop_token(&ctxt);
     }
     g_free(ctxt.current);
-    g_queue_free(ctxt.buf);
 
     return result;
 }
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 954bf9d468..9210281a65 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -22,17 +22,12 @@
 #define MAX_TOKEN_COUNT (2ULL << 20)
 #define MAX_NESTING (1 << 10)
 
-static void json_message_free_token(void *token, void *opaque)
-{
-    g_free(token);
-}
-
 static void json_message_free_tokens(JSONMessageParser *parser)
 {
-    if (parser->tokens) {
-        g_queue_foreach(parser->tokens, json_message_free_token, NULL);
-        g_queue_free(parser->tokens);
-        parser->tokens = NULL;
+    JSONToken *token;
+
+    while ((token = g_queue_pop_head(&parser->tokens))) {
+        g_free(token);
     }
 }
 
@@ -61,11 +56,10 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
         error_setg(&err, "JSON parse error, stray '%s'", input->str);
         goto out_emit;
     case JSON_END_OF_INPUT:
-        if (g_queue_is_empty(parser->tokens)) {
+        if (g_queue_is_empty(&parser->tokens)) {
             return;
         }
-        json = json_parser_parse(parser->tokens, parser->ap, &err);
-        parser->tokens = NULL;
+        json = json_parser_parse(&parser->tokens, parser->ap, &err);
         goto out_emit;
     default:
         break;
@@ -79,7 +73,7 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
         error_setg(&err, "JSON token size limit exceeded");
         goto out_emit;
     }
-    if (g_queue_get_length(parser->tokens) + 1 > MAX_TOKEN_COUNT) {
+    if (g_queue_get_length(&parser->tokens) + 1 > MAX_TOKEN_COUNT) {
         error_setg(&err, "JSON token count limit exceeded");
         goto out_emit;
     }
@@ -97,21 +91,19 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
 
     parser->token_size += input->len;
 
-    g_queue_push_tail(parser->tokens, token);
+    g_queue_push_tail(&parser->tokens, token);
 
     if ((parser->brace_count > 0 || parser->bracket_count > 0)
         && parser->bracket_count >= 0 && parser->bracket_count >= 0) {
         return;
     }
 
-    json = json_parser_parse(parser->tokens, parser->ap, &err);
-    parser->tokens = NULL;
+    json = json_parser_parse(&parser->tokens, parser->ap, &err);
 
 out_emit:
     parser->brace_count = 0;
     parser->bracket_count = 0;
     json_message_free_tokens(parser);
-    parser->tokens = g_queue_new();
     parser->token_size = 0;
     parser->emit(parser->opaque, json, err);
 }
@@ -126,7 +118,7 @@ void json_message_parser_init(JSONMessageParser *parser,
     parser->ap = ap;
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    parser->tokens = g_queue_new();
+    g_queue_init(&parser->tokens);
     parser->token_size = 0;
 
     json_lexer_init(&parser->lexer, !!ap);
@@ -141,7 +133,7 @@ void json_message_parser_feed(JSONMessageParser *parser,
 void json_message_parser_flush(JSONMessageParser *parser)
 {
     json_lexer_flush(&parser->lexer);
-    assert(g_queue_is_empty(parser->tokens));
+    assert(g_queue_is_empty(&parser->tokens));
 }
 
 void json_message_parser_destroy(JSONMessageParser *parser)
-- 
cgit v1.2.3-55-g7522


From abe7c2067c21a89c6fd4cf2d8ba2fa37160d1d55 Mon Sep 17 00:00:00 2001
From: Markus Armbruster
Date: Thu, 23 Aug 2018 18:40:18 +0200
Subject: json: Make JSONToken opaque outside json-parser.c

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180823164025.12553-52-armbru@redhat.com>
---
 include/qapi/qmp/json-parser.h   |  4 ++++
 include/qapi/qmp/json-streamer.h |  7 -------
 qobject/json-parser.c            | 19 +++++++++++++++++++
 qobject/json-streamer.c          |  8 +-------
 4 files changed, 24 insertions(+), 14 deletions(-)

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h
index a34209db7a..21b23d7bec 100644
--- a/include/qapi/qmp/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -15,7 +15,11 @@
 #define QEMU_JSON_PARSER_H
 
 #include "qemu-common.h"
+#include "qapi/qmp/json-lexer.h"
 
+typedef struct JSONToken JSONToken;
+
+JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr);
 QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp);
 
 #endif
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index d1d7fe2595..29950ac37c 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -16,13 +16,6 @@
 
 #include "qapi/qmp/json-lexer.h"
 
-typedef struct JSONToken {
-    int type;
-    int x;
-    int y;
-    char str[];
-} JSONToken;
-
 typedef struct JSONMessageParser
 {
     void (*emit)(void *opaque, QObject *json, Error *err);
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index e9a9f937f3..a247875f14 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -26,6 +26,13 @@
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-streamer.h"
 
+struct JSONToken {
+    JSONTokenType type;
+    int x;
+    int y;
+    char str[];
+};
+
 typedef struct JSONParserContext
 {
     Error *err;
@@ -538,6 +545,18 @@ static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
     }
 }
 
+JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr)
+{
+    JSONToken *token = g_malloc(sizeof(JSONToken) + tokstr->len + 1);
+
+    token->type = type;
+    memcpy(token->str, tokstr->str, tokstr->len);
+    token->str[tokstr->len] = 0;
+    token->x = x;
+    token->y = y;
+    return token;
+}
+
 QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp)
 {
     JSONParserContext ctxt = { .buf = tokens };
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 9210281a65..467bc29413 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -82,13 +82,7 @@ void json_message_process_token(JSONLexer *lexer, GString *input,
         goto out_emit;
     }
 
-    token = g_malloc(sizeof(JSONToken) + input->len + 1);
-    token->type = type;
-    memcpy(token->str, input->str, input->len);
-    token->str[input->len] = 0;
-    token->x = x;
-    token->y = y;
-
+    token = json_token(type, x, y, input);
     parser->token_size += input->len;
 
     g_queue_push_tail(&parser->tokens, token);
-- 
cgit v1.2.3-55-g7522


From 86cdf9ec8dec2763702cc52fa412d108a5dc9608 Mon Sep 17 00:00:00 2001
From: Markus Armbruster
Date: Thu, 23 Aug 2018 18:40:20 +0200
Subject: json: Clean up headers

The JSON parser has three public headers, json-lexer.h, json-parser.h,
json-streamer.h.  They all contain stuff that is of no interest
outside qobject/json-*.c.

Collect the public interface in include/qapi/qmp/json-parser.h, and
everything else in qobject/json-parser-int.h.

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180823164025.12553-54-armbru@redhat.com>
---
 include/qapi/qmp/json-lexer.h    | 50 -------------------------------------
 include/qapi/qmp/json-parser.h   | 36 +++++++++++++++++++++------
 include/qapi/qmp/json-streamer.h | 46 ----------------------------------
 monitor.c                        |  2 +-
 qga/main.c                       |  2 +-
 qobject/json-lexer.c             |  3 +--
 qobject/json-parser-int.h        | 54 ++++++++++++++++++++++++++++++++++++++++
 qobject/json-parser.c            |  4 +--
 qobject/json-streamer.c          |  4 +--
 qobject/qjson.c                  |  2 +-
 tests/libqtest.c                 |  2 +-
 11 files changed, 90 insertions(+), 115 deletions(-)
 delete mode 100644 include/qapi/qmp/json-lexer.h
 delete mode 100644 include/qapi/qmp/json-streamer.h
 create mode 100644 qobject/json-parser-int.h

(limited to 'include/qapi/qmp/json-streamer.h')

diff --git a/include/qapi/qmp/json-lexer.h b/include/qapi/qmp/json-lexer.h
deleted file mode 100644
index 508fc7bdaf..0000000000
--- a/include/qapi/qmp/json-lexer.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * JSON lexer
- *
- * Copyright IBM, Corp. 2009
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QEMU_JSON_LEXER_H
-#define QEMU_JSON_LEXER_H
-
-
-typedef enum json_token_type {
-    JSON_MIN = 100,
-    JSON_LCURLY = JSON_MIN,
-    JSON_RCURLY,
-    JSON_LSQUARE,
-    JSON_RSQUARE,
-    JSON_COLON,
-    JSON_COMMA,
-    JSON_INTEGER,
-    JSON_FLOAT,
-    JSON_KEYWORD,
-    JSON_STRING,
-    JSON_INTERP,
-    JSON_SKIP,
-    JSON_ERROR,
-    JSON_END_OF_INPUT,
-} JSONTokenType;
-
-typedef struct JSONLexer {
-    int start_state, state;
-    GString *token;
-    int x, y;
-} JSONLexer;
-
-void json_lexer_init(JSONLexer *lexer, bool enable_interpolation);
-
-void json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size);
-
-void json_lexer_flush(JSONLexer *lexer);
-
-void json_lexer_destroy(JSONLexer *lexer);
-
-#endif
diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h
index 55f75954c3..7345a9bd5c 100644
--- a/include/qapi/qmp/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -1,5 +1,5 @@
 /*
- * JSON Parser 
+ * JSON Parser
  *
  * Copyright IBM, Corp. 2009
  *
@@ -11,14 +11,36 @@
  *
  */
 
-#ifndef QEMU_JSON_PARSER_H
-#define QEMU_JSON_PARSER_H
+#ifndef QAPI_QMP_JSON_PARSER_H
+#define QAPI_QMP_JSON_PARSER_H
 
-#include "qapi/qmp/json-lexer.h"
+typedef struct JSONLexer {
+    int start_state, state;
+    GString *token;
+    int x, y;
+} JSONLexer;
 
-typedef struct JSONToken JSONToken;
+typedef struct JSONMessageParser {
+    void (*emit)(void *opaque, QObject *json, Error *err);
+    void *opaque;
+    va_list *ap;
+    JSONLexer lexer;
+    int brace_count;
+    int bracket_count;
+    GQueue tokens;
+    uint64_t token_size;
+} JSONMessageParser;
 
-JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr);
-QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp);
+void json_message_parser_init(JSONMessageParser *parser,
+                              void (*emit)(void *opaque, QObject *json,
+                                           Error *err),
+                              void *opaque, va_list *ap);
+
+void json_message_parser_feed(JSONMessageParser *parser,
+                             const char *buffer, size_t size);
+
+void json_message_parser_flush(JSONMessageParser *parser);
+
+void json_message_parser_destroy(JSONMessageParser *parser);
 
 #endif
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
deleted file mode 100644
index 29950ac37c..0000000000
--- a/include/qapi/qmp/json-streamer.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * JSON streaming support
- *
- * Copyright IBM, Corp. 2009
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QEMU_JSON_STREAMER_H
-#define QEMU_JSON_STREAMER_H
-
-#include "qapi/qmp/json-lexer.h"
-
-typedef struct JSONMessageParser
-{
-    void (*emit)(void *opaque, QObject *json, Error *err);
-    void *opaque;
-    va_list *ap;
-    JSONLexer lexer;
-    int brace_count;
-    int bracket_count;
-    GQueue tokens;
-    uint64_t token_size;
-} JSONMessageParser;
-
-void json_message_process_token(JSONLexer *lexer, GString *input,
-                                JSONTokenType type, int x, int y);
-
-void json_message_parser_init(JSONMessageParser *parser,
-                              void (*emit)(void *opaque, QObject *json,
-                                           Error *err),
-                              void *opaque, va_list *ap);
-
-void json_message_parser_feed(JSONMessageParser *parser,
-                             const char *buffer, size_t size);
-
-void json_message_parser_flush(JSONMessageParser *parser);
-
-void json_message_parser_destroy(JSONMessageParser *parser);
-
-#endif
diff --git a/monitor.c b/monitor.c
index 3dbdcb5190..021c11b1bf 100644
--- a/monitor.c
+++ b/monitor.c
@@ -58,7 +58,7 @@
 #include "qapi/qmp/qnum.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
-#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qlist.h"
 #include "qom/object_interfaces.h"
 #include "trace-root.h"
diff --git a/qga/main.c b/qga/main.c
index b74e1241ef..6d70242d05 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -18,7 +18,7 @@
 #include <syslog.h>
 #include <sys/wait.h>
 #endif
-#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qstring.h"
diff --git a/qobject/json-lexer.c b/qobject/json-lexer.c
index 06ec67dc45..e1745a3d95 100644
--- a/qobject/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -12,8 +12,7 @@
  */
 
 #include "qemu/osdep.h"
-#include "qapi/qmp/json-lexer.h"
-#include "qapi/qmp/json-streamer.h"
+#include "json-parser-int.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
diff --git a/qobject/json-parser-int.h b/qobject/json-parser-int.h
new file mode 100644
index 0000000000..ceaa890ec6
--- /dev/null
+++ b/qobject/json-parser-int.h
@@ -0,0 +1,54 @@
+/*
+ * JSON Parser
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef JSON_PARSER_INT_H
+#define JSON_PARSER_INT_H
+
+#include "qapi/qmp/json-parser.h"
+
+
+typedef enum json_token_type {
+    JSON_MIN = 100,
+    JSON_LCURLY = JSON_MIN,
+    JSON_RCURLY,
+    JSON_LSQUARE,
+    JSON_RSQUARE,
+    JSON_COLON,
+    JSON_COMMA,
+    JSON_INTEGER,
+    JSON_FLOAT,
+    JSON_KEYWORD,
+    JSON_STRING,
+    JSON_INTERP,
+    JSON_SKIP,
+    JSON_ERROR,
+    JSON_END_OF_INPUT,
+} JSONTokenType;
+
+typedef struct JSONToken JSONToken;
+
+/* json-lexer.c */
+void json_lexer_init(JSONLexer *lexer, bool enable_interpolation);
+void json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size);
+void json_lexer_flush(JSONLexer *lexer);
+void json_lexer_destroy(JSONLexer *lexer);
+
+/* json-streamer.c */
+void json_message_process_token(JSONLexer *lexer, GString *input,
+                                JSONTokenType type, int x, int y);
+
+/* json-parser.c */
+JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr);
+QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp);
+
+#endif
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index a247875f14..7449684f1c 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -22,9 +22,7 @@
 #include "qapi/qmp/qnull.h"
 #include "qapi/qmp/qnum.h"
 #include "qapi/qmp/qstring.h"
-#include "qapi/qmp/json-parser.h"
-#include "qapi/qmp/json-lexer.h"
-#include "qapi/qmp/json-streamer.h"
+#include "json-parser-int.h"
 
 struct JSONToken {
     JSONTokenType type;
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index da53e770e9..47dd7ea576 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -13,9 +13,7 @@
 
 #include "qemu/osdep.h"
 #include "qapi/error.h"
-#include "qapi/qmp/json-lexer.h"
-#include "qapi/qmp/json-parser.h"
-#include "qapi/qmp/json-streamer.h"
+#include "json-parser-int.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 #define MAX_TOKEN_COUNT (2ULL << 20)
diff --git a/qobject/qjson.c b/qobject/qjson.c
index b9ccae2c2a..db36101f3b 100644
--- a/qobject/qjson.c
+++ b/qobject/qjson.c
@@ -13,7 +13,7 @@
 
 #include "qemu/osdep.h"
 #include "qapi/error.h"
-#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qdict.h"
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 5973a67652..d635c5bea0 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -24,7 +24,7 @@
 #include "qemu-common.h"
 #include "qemu/cutils.h"
 #include "qapi/error.h"
-#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qlist.h"
-- 
cgit v1.2.3-55-g7522