From 62815d85aed71eff7b6c3a524705180fb04f5d30 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:01 +0200 Subject: json: Redesign the callback to consume JSON values The classical way to structure parser and lexer is to have the client call the parser to get an abstract syntax tree, the parser call the lexer to get the next token, and the lexer call some function to get input characters. Another way to structure them would be to have the client feed characters to the lexer, the lexer feed tokens to the parser, and the parser feed abstract syntax trees to some callback provided by the client. This way is more easily integrated into an event loop that dispatches input characters as they arrive. Our JSON parser is kind of between the two. The lexer feeds tokens to a "streamer" instead of a real parser. The streamer accumulates tokens until it got the sequence of tokens that comprise a single JSON value (it counts curly braces and square brackets to decide). It feeds those token sequences to a callback provided by the client. The callback passes each token sequence to the parser, and gets back an abstract syntax tree. I figure it was done that way to make a straightforward recursive descent parser possible. "Get next token" becomes "pop the first token off the token sequence". Drawback: we need to store a complete token sequence. Each token eats 13 + input characters + malloc overhead bytes. Observations: 1. This is not the only way to use recursive descent. If we replaced "get next token" by a coroutine yield, we could do without a streamer. 2. The lexer reports errors by passing a JSON_ERROR token to the streamer. This communicates the offending input characters and their location, but no more. 3. The streamer reports errors by passing a null token sequence to the callback. The (already poor) lexical error information is thrown away. 4. Having the callback receive a token sequence duplicates the code to convert token sequence to abstract syntax tree in every callback. 5. Known bug: the streamer silently drops incomplete token sequences. This commit rectifies 4. by lifting the call of the parser from the callbacks into the streamer. Later commits will address 3. and 5. The lifting removes a bug from qjson.c's parse_json(): it passed a pointer to a non-null Error * in certain cases, as demonstrated by check-qjson.c. json_parser_parse() is now unused. It's a stupid wrapper around json_parser_parse_err(). Drop it, and rename json_parser_parse_err() to json_parser_parse(). Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-35-armbru@redhat.com> --- qapi/qmp-dispatch.c | 1 - 1 file changed, 1 deletion(-) (limited to 'qapi') diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c index 6f2d466596..d8da1a62de 100644 --- a/qapi/qmp-dispatch.c +++ b/qapi/qmp-dispatch.c @@ -14,7 +14,6 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "qapi/qmp/dispatch.h" -#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/qbool.h" -- cgit v1.2.3-55-g7522 From dd98e8481992741a6b5ec0bdfcee05c1c8f602d6 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:14 +0200 Subject: qjson: Have qobject_from_json() & friends reject empty and blank The last case where qobject_from_json() & friends return null without setting an error is empty or blank input. Callers: * block.c's parse_json_protocol() reports "Could not parse the JSON options". It's marked as a work-around, because it also covered actual bugs, but they got fixed in the previous few commits. * qobject_input_visitor_new_str() reports "JSON parse error". Also marked as work-around. The recent fixes have made this unreachable, because it currently gets called only for input starting with '{'. * check-qjson.c's empty_input() and blank_input() demonstrate the behavior. * The other callers are not affected since they only pass input with exactly one JSON value or, in the case of negative tests, one error. Fail with "Expecting a JSON value" instead of returning null, and simplify callers. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-48-armbru@redhat.com> --- block.c | 5 ----- qapi/qobject-input-visitor.c | 5 ----- qobject/qjson.c | 4 ++++ tests/check-qjson.c | 12 ++++++++++-- 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'qapi') diff --git a/block.c b/block.c index 6161dbe3eb..0dbb1fcc7b 100644 --- a/block.c +++ b/block.c @@ -1478,11 +1478,6 @@ static QDict *parse_json_filename(const char *filename, Error **errp) options_obj = qobject_from_json(filename, errp); if (!options_obj) { - /* Work around qobject_from_json() lossage TODO fix that */ - if (errp && !*errp) { - error_setg(errp, "Could not parse the JSON options"); - return NULL; - } error_prepend(errp, "Could not parse the JSON options: "); return NULL; } diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c index da57f4cc24..3e88b27f9e 100644 --- a/qapi/qobject-input-visitor.c +++ b/qapi/qobject-input-visitor.c @@ -725,11 +725,6 @@ Visitor *qobject_input_visitor_new_str(const char *str, if (is_json) { obj = qobject_from_json(str, errp); if (!obj) { - /* Work around qobject_from_json() lossage TODO fix that */ - if (errp && !*errp) { - error_setg(errp, "JSON parse error"); - return NULL; - } return NULL; } args = qobject_to(QDict, obj); diff --git a/qobject/qjson.c b/qobject/qjson.c index 7f69036487..b9ccae2c2a 100644 --- a/qobject/qjson.c +++ b/qobject/qjson.c @@ -70,6 +70,10 @@ static QObject *qobject_from_jsonv(const char *string, va_list *ap, json_message_parser_flush(&state.parser); json_message_parser_destroy(&state.parser); + if (!state.result && !state.err) { + error_setg(&state.err, "Expecting a JSON value"); + } + error_propagate(errp, state.err); return state.result; } diff --git a/tests/check-qjson.c b/tests/check-qjson.c index 0ca4b3c823..936258ddd4 100644 --- a/tests/check-qjson.c +++ b/tests/check-qjson.c @@ -1291,13 +1291,21 @@ static void simple_interpolation(void) static void empty_input(void) { - QObject *obj = qobject_from_json("", &error_abort); + Error *err = NULL; + QObject *obj; + + obj = qobject_from_json("", &err); + error_free_or_abort(&err); g_assert(obj == NULL); } static void blank_input(void) { - QObject *obj = qobject_from_json("\n ", &error_abort); + Error *err = NULL; + QObject *obj; + + obj = qobject_from_json("\n ", &err); + error_free_or_abort(&err); g_assert(obj == NULL); } -- cgit v1.2.3-55-g7522 From 37aded92c27d0e56cd27f1c29494fc9f8c873cdd Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:25 +0200 Subject: json: Update references to RFC 7159 to RFC 8259 RFC 8259 (December 2017) obsoletes RFC 7159 (March 2014). Signed-off-by: Markus Armbruster Message-Id: <20180823164025.12553-59-armbru@redhat.com> Reviewed-by: Eric Blake --- include/qapi/qmp/qnum.h | 2 +- qapi/introspect.json | 2 +- qobject/json-parser.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'qapi') diff --git a/include/qapi/qmp/qnum.h b/include/qapi/qmp/qnum.h index 45bf02a036..bbae0a5ec8 100644 --- a/include/qapi/qmp/qnum.h +++ b/include/qapi/qmp/qnum.h @@ -25,7 +25,7 @@ typedef enum { /* * QNum encapsulates how our dialect of JSON fills in the blanks left - * by the JSON specification (RFC 7159) regarding numbers. + * by the JSON specification (RFC 8259) regarding numbers. * * Conceptually, we treat number as an abstract type with three * concrete subtypes: floating-point, signed integer, unsigned diff --git a/qapi/introspect.json b/qapi/introspect.json index 137b39b992..3d22166b2b 100644 --- a/qapi/introspect.json +++ b/qapi/introspect.json @@ -120,7 +120,7 @@ ## # @JSONType: # -# The four primitive and two structured types according to RFC 7159 +# The four primitive and two structured types according to RFC 8259 # section 1, plus 'int' (split off 'number'), plus the obvious top # type 'value'. # diff --git a/qobject/json-parser.c b/qobject/json-parser.c index 3318b8dad0..5a840dfd86 100644 --- a/qobject/json-parser.c +++ b/qobject/json-parser.c @@ -516,7 +516,7 @@ static QObject *parse_literal(JSONParserContext *ctxt) } case JSON_FLOAT: /* FIXME dependent on locale; a pervasive issue in QEMU */ - /* FIXME our lexer matches RFC 7159 in forbidding Inf or NaN, + /* FIXME our lexer matches RFC 8259 in forbidding Inf or NaN, * but those might be useful extensions beyond JSON */ return QOBJECT(qnum_from_double(strtod(token->str, NULL))); default: -- cgit v1.2.3-55-g7522