34b321
From 7bef5fab7d59aa9a6a1eb6ea747ba04811bc01e2 Mon Sep 17 00:00:00 2001
34b321
From: Markus Armbruster <armbru@redhat.com>
34b321
Date: Wed, 27 Jul 2016 07:35:10 +0200
34b321
Subject: [PATCH 12/16] qjson: surprise, allocating 6 QObjects per token is
34b321
 expensive
34b321
34b321
RH-Author: Markus Armbruster <armbru@redhat.com>
34b321
Message-id: <1469604913-12442-14-git-send-email-armbru@redhat.com>
34b321
Patchwork-id: 71475
34b321
O-Subject: [RHEL-7.3 qemu-kvm PATCH v2 12/15] qjson: surprise, allocating 6 QObjects per token is expensive
34b321
Bugzilla: 1276036
34b321
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
34b321
RH-Acked-by: John Snow <jsnow@redhat.com>
34b321
RH-Acked-by: Paolo Bonzini <pbonzini@redhat.com>
34b321
34b321
From: Paolo Bonzini <pbonzini@redhat.com>
34b321
34b321
Replace the contents of the tokens GQueue with a simple struct.  This cuts
34b321
the amount of memory allocated by tests/check-qjson from ~500MB to ~20MB,
34b321
and the execution time from 600ms to 80ms on my laptop.  Still a lot (some
34b321
could be saved by using an intrusive list, such as QSIMPLEQ, instead of
34b321
the GQueue), but the savings are already massive and the right thing to
34b321
do would probably be to get rid of json-streamer completely.
34b321
34b321
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
34b321
Message-Id: <1448300659-23559-5-git-send-email-pbonzini@redhat.com>
34b321
[Straightforwardly rebased on my patches]
34b321
Signed-off-by: Markus Armbruster <armbru@redhat.com>
34b321
Reviewed-by: Eric Blake <eblake@redhat.com>
34b321
(cherry picked from commit 9bada8971173345ceb37ed1a47b00a01a4dd48cf)
34b321
34b321
Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>
34b321
34b321
Conflicts:
34b321
	qobject/json-parser.c
34b321
34b321
Straighforward conflicts because lacking commit fc48ffc "qobject: Use
34b321
'bool' for qbool", we still use qbool_from_int(), and we lack commit
34b321
e549e71 "json-parser: Accept 'null' in QMP".
34b321
34b321
Signed-off-by: Markus Armbruster <armbru@redhat.com>
34b321
---
34b321
 include/qapi/qmp/json-streamer.h |   7 +++
34b321
 qobject/json-parser.c            | 113 ++++++++++++++++-----------------------
34b321
 qobject/json-streamer.c          |  19 +++----
34b321
 3 files changed, 62 insertions(+), 77 deletions(-)
34b321
34b321
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
34b321
index e9f2937..09b3d3e 100644
34b321
--- a/include/qapi/qmp/json-streamer.h
34b321
+++ b/include/qapi/qmp/json-streamer.h
34b321
@@ -18,6 +18,13 @@
34b321
 #include "glib-compat.h"
34b321
 #include "qapi/qmp/json-lexer.h"
34b321
 
34b321
+typedef struct JSONToken {
34b321
+    int type;
34b321
+    int x;
34b321
+    int y;
34b321
+    char str[];
34b321
+} JSONToken;
34b321
+
34b321
 typedef struct JSONMessageParser
34b321
 {
34b321
     void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
34b321
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
34b321
index 6e5e257..944e1a1 100644
34b321
--- a/qobject/json-parser.c
34b321
+++ b/qobject/json-parser.c
34b321
@@ -23,11 +23,12 @@
34b321
 #include "qapi/qmp/json-parser.h"
34b321
 #include "qapi/qmp/json-lexer.h"
34b321
 #include "qapi/qmp/qerror.h"
34b321
+#include "qapi/qmp/json-streamer.h"
34b321
 
34b321
 typedef struct JSONParserContext
34b321
 {
34b321
     Error *err;
34b321
-    QObject *current;
34b321
+    JSONToken *current;
34b321
     GQueue *buf;
34b321
 } JSONParserContext;
34b321
 
34b321
@@ -45,27 +46,10 @@ typedef struct JSONParserContext
34b321
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
34b321
 
34b321
 /**
34b321
- * Token manipulators
34b321
- *
34b321
- * tokens are dictionaries that contain a type, a string value, and geometry information
34b321
- * about a token identified by the lexer.  These are routines that make working with
34b321
- * these objects a bit easier.
34b321
- */
34b321
-static const char *token_get_value(QObject *obj)
34b321
-{
34b321
-    return qdict_get_str(qobject_to_qdict(obj), "token");
34b321
-}
34b321
-
34b321
-static JSONTokenType token_get_type(QObject *obj)
34b321
-{
34b321
-    return qdict_get_int(qobject_to_qdict(obj), "type");
34b321
-}
34b321
-
34b321
-/**
34b321
  * Error handler
34b321
  */
34b321
 static void GCC_FMT_ATTR(3, 4) parse_error(JSONParserContext *ctxt,
34b321
-                                           QObject *token, const char *msg, ...)
34b321
+                                           JSONToken *token, const char *msg, ...)
34b321
 {
34b321
     va_list ap;
34b321
     char message[1024];
34b321
@@ -143,9 +127,10 @@ static int hex2decimal(char ch)
34b321
  *      \t
34b321
  *      \u four-hex-digits 
34b321
  */
34b321
-static QString *qstring_from_escaped_str(JSONParserContext *ctxt, QObject *token)
34b321
+static QString *qstring_from_escaped_str(JSONParserContext *ctxt,
34b321
+                                         JSONToken *token)
34b321
 {
34b321
-    const char *ptr = token_get_value(token);
34b321
+    const char *ptr = token->str;
34b321
     QString *str;
34b321
     int double_quote = 1;
34b321
 
34b321
@@ -241,19 +226,19 @@ out:
34b321
     return NULL;
34b321
 }
34b321
 
34b321
-/* Note: unless the token object returned by parser_context_peek_token
34b321
- * or parser_context_pop_token is explicitly incref'd, it will be
34b321
- * deleted as soon as parser_context_pop_token is called again.
34b321
+/* Note: the token object returned by parser_context_peek_token or
34b321
+ * parser_context_pop_token is deleted as soon as parser_context_pop_token
34b321
+ * is called again.
34b321
  */
34b321
-static QObject *parser_context_pop_token(JSONParserContext *ctxt)
34b321
+static JSONToken *parser_context_pop_token(JSONParserContext *ctxt)
34b321
 {
34b321
-    qobject_decref(ctxt->current);
34b321
+    g_free(ctxt->current);
34b321
     assert(!g_queue_is_empty(ctxt->buf));
34b321
     ctxt->current = g_queue_pop_head(ctxt->buf);
34b321
     return ctxt->current;
34b321
 }
34b321
 
34b321
-static QObject *parser_context_peek_token(JSONParserContext *ctxt)
34b321
+static JSONToken *parser_context_peek_token(JSONParserContext *ctxt)
34b321
 {
34b321
     assert(!g_queue_is_empty(ctxt->buf));
34b321
     return g_queue_peek_head(ctxt->buf);
34b321
@@ -280,7 +265,7 @@ static void parser_context_free(JSONParserContext *ctxt)
34b321
         while (!g_queue_is_empty(ctxt->buf)) {
34b321
             parser_context_pop_token(ctxt);
34b321
         }
34b321
-        qobject_decref(ctxt->current);
34b321
+        g_free(ctxt->current);
34b321
         g_queue_free(ctxt->buf);
34b321
         g_free(ctxt);
34b321
     }
34b321
@@ -291,7 +276,8 @@ static void parser_context_free(JSONParserContext *ctxt)
34b321
  */
34b321
 static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
34b321
 {
34b321
-    QObject *key = NULL, *token = NULL, *value, *peek;
34b321
+    QObject *key = NULL, *value;
34b321
+    JSONToken *peek, *token;
34b321
 
34b321
     peek = parser_context_peek_token(ctxt);
34b321
     if (peek == NULL) {
34b321
@@ -311,7 +297,7 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
34b321
         goto out;
34b321
     }
34b321
 
34b321
-    if (token_get_type(token) != JSON_COLON) {
34b321
+    if (token->type != JSON_COLON) {
34b321
         parse_error(ctxt, token, "missing : in object pair");
34b321
         goto out;
34b321
     }
34b321
@@ -337,10 +323,10 @@ out:
34b321
 static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
34b321
 {
34b321
     QDict *dict = NULL;
34b321
-    QObject *token, *peek;
34b321
+    JSONToken *token, *peek;
34b321
 
34b321
     token = parser_context_pop_token(ctxt);
34b321
-    assert(token && token_get_type(token) == JSON_LCURLY);
34b321
+    assert(token && token->type == JSON_LCURLY);
34b321
 
34b321
     dict = qdict_new();
34b321
 
34b321
@@ -350,7 +336,7 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
34b321
         goto out;
34b321
     }
34b321
 
34b321
-    if (token_get_type(peek) != JSON_RCURLY) {
34b321
+    if (peek->type != JSON_RCURLY) {
34b321
         if (parse_pair(ctxt, dict, ap) == -1) {
34b321
             goto out;
34b321
         }
34b321
@@ -361,8 +347,8 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
34b321
             goto out;
34b321
         }
34b321
 
34b321
-        while (token_get_type(token) != JSON_RCURLY) {
34b321
-            if (token_get_type(token) != JSON_COMMA) {
34b321
+        while (token->type != JSON_RCURLY) {
34b321
+            if (token->type != JSON_COMMA) {
34b321
                 parse_error(ctxt, token, "expected separator in dict");
34b321
                 goto out;
34b321
             }
34b321
@@ -391,10 +377,10 @@ out:
34b321
 static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
34b321
 {
34b321
     QList *list = NULL;
34b321
-    QObject *token, *peek;
34b321
+    JSONToken *token, *peek;
34b321
 
34b321
     token = parser_context_pop_token(ctxt);
34b321
-    assert(token && token_get_type(token) == JSON_LSQUARE);
34b321
+    assert(token && token->type == JSON_LSQUARE);
34b321
 
34b321
     list = qlist_new();
34b321
 
34b321
@@ -404,7 +390,7 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
34b321
         goto out;
34b321
     }
34b321
 
34b321
-    if (token_get_type(peek) != JSON_RSQUARE) {
34b321
+    if (peek->type != JSON_RSQUARE) {
34b321
         QObject *obj;
34b321
 
34b321
         obj = parse_value(ctxt, ap);
34b321
@@ -421,8 +407,8 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
34b321
             goto out;
34b321
         }
34b321
 
34b321
-        while (token_get_type(token) != JSON_RSQUARE) {
34b321
-            if (token_get_type(token) != JSON_COMMA) {
34b321
+        while (token->type != JSON_RSQUARE) {
34b321
+            if (token->type != JSON_COMMA) {
34b321
                 parse_error(ctxt, token, "expected separator in list");
34b321
                 goto out;
34b321
             }
34b321
@@ -454,49 +440,45 @@ out:
34b321
 
34b321
 static QObject *parse_keyword(JSONParserContext *ctxt)
34b321
 {
34b321
-    QObject *token;
34b321
-    const char *val;
34b321
+    JSONToken *token;
34b321
 
34b321
     token = parser_context_pop_token(ctxt);
34b321
-    assert(token && token_get_type(token) == JSON_KEYWORD);
34b321
-    val = token_get_value(token);
34b321
+    assert(token && token->type == JSON_KEYWORD);
34b321
 
34b321
-    if (!strcmp(val, "true")) {
34b321
+    if (!strcmp(token->str, "true")) {
34b321
         return QOBJECT(qbool_from_int(true));
34b321
-    } else if (!strcmp(val, "false")) {
34b321
+    } else if (!strcmp(token->str, "false")) {
34b321
         return QOBJECT(qbool_from_int(false));
34b321
     }
34b321
-    parse_error(ctxt, token, "invalid keyword '%s'", val);
34b321
+    parse_error(ctxt, token, "invalid keyword '%s'", token->str);
34b321
     return NULL;
34b321
 }
34b321
 
34b321
 static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
34b321
 {
34b321
-    QObject *token;
34b321
-    const char *val;
34b321
+    JSONToken *token;
34b321
 
34b321
     if (ap == NULL) {
34b321
         return NULL;
34b321
     }
34b321
 
34b321
     token = parser_context_pop_token(ctxt);
34b321
-    assert(token && token_get_type(token) == JSON_ESCAPE);
34b321
-    val = token_get_value(token);
34b321
+    assert(token && token->type == JSON_ESCAPE);
34b321
 
34b321
-    if (!strcmp(val, "%p")) {
34b321
+    if (!strcmp(token->str, "%p")) {
34b321
         return va_arg(*ap, QObject *);
34b321
-    } else if (!strcmp(val, "%i")) {
34b321
+    } else if (!strcmp(token->str, "%i")) {
34b321
         return QOBJECT(qbool_from_int(va_arg(*ap, int)));
34b321
-    } else if (!strcmp(val, "%d")) {
34b321
+    } else if (!strcmp(token->str, "%d")) {
34b321
         return QOBJECT(qint_from_int(va_arg(*ap, int)));
34b321
-    } else if (!strcmp(val, "%ld")) {
34b321
+    } else if (!strcmp(token->str, "%ld")) {
34b321
         return QOBJECT(qint_from_int(va_arg(*ap, long)));
34b321
-    } else if (!strcmp(val, "%lld") ||
34b321
-               !strcmp(val, "%I64d")) {
34b321
+    } else if (!strcmp(token->str, "%lld") ||
34b321
+               !strcmp(token->str, "%I64d")) {
34b321
         return QOBJECT(qint_from_int(va_arg(*ap, long long)));
34b321
-    } else if (!strcmp(val, "%s")) {
34b321
+    } else if (!strcmp(token->str, "%s")) {
34b321
         return QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
34b321
-    } else if (!strcmp(val, "%f")) {
34b321
+    } else if (!strcmp(token->str, "%f")) {
34b321
         return QOBJECT(qfloat_from_double(va_arg(*ap, double)));
34b321
     }
34b321
     return NULL;
34b321
@@ -504,12 +486,12 @@ static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
34b321
 
34b321
 static QObject *parse_literal(JSONParserContext *ctxt)
34b321
 {
34b321
-    QObject *token;
34b321
+    JSONToken *token;
34b321
 
34b321
     token = parser_context_pop_token(ctxt);
34b321
     assert(token);
34b321
 
34b321
-    switch (token_get_type(token)) {
34b321
+    switch (token->type) {
34b321
     case JSON_STRING:
34b321
         return QOBJECT(qstring_from_escaped_str(ctxt, token));
34b321
     case JSON_INTEGER: {
34b321
@@ -528,7 +510,7 @@ static QObject *parse_literal(JSONParserContext *ctxt)
34b321
         int64_t value;
34b321
 
34b321
         errno = 0; /* strtoll doesn't set errno on success */
34b321
-        value = strtoll(token_get_value(token), NULL, 10);
34b321
+        value = strtoll(token->str, NULL, 10);
34b321
         if (errno != ERANGE) {
34b321
             return QOBJECT(qint_from_int(value));
34b321
         }
34b321
@@ -536,8 +518,7 @@ static QObject *parse_literal(JSONParserContext *ctxt)
34b321
     }
34b321
     case JSON_FLOAT:
34b321
         /* FIXME dependent on locale */
34b321
-        return QOBJECT(qfloat_from_double(strtod(token_get_value(token),
34b321
-                                                 NULL)));
34b321
+        return QOBJECT(qfloat_from_double(strtod(token->str, NULL)));
34b321
     default:
34b321
         abort();
34b321
     }
34b321
@@ -545,7 +526,7 @@ static QObject *parse_literal(JSONParserContext *ctxt)
34b321
 
34b321
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
34b321
 {
34b321
-    QObject *token;
34b321
+    JSONToken *token;
34b321
 
34b321
     token = parser_context_peek_token(ctxt);
34b321
     if (token == NULL) {
34b321
@@ -553,7 +534,7 @@ static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
34b321
         return NULL;
34b321
     }
34b321
 
34b321
-    switch (token_get_type(token)) {
34b321
+    switch (token->type) {
34b321
     case JSON_LCURLY:
34b321
         return parse_object(ctxt, ap);
34b321
     case JSON_LSQUARE:
34b321
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
34b321
index f7a3e78..e87230d 100644
34b321
--- a/qobject/json-streamer.c
34b321
+++ b/qobject/json-streamer.c
34b321
@@ -11,10 +11,6 @@
34b321
  *
34b321
  */
34b321
 
34b321
-#include "qapi/qmp/qlist.h"
34b321
-#include "qapi/qmp/qstring.h"
34b321
-#include "qapi/qmp/qint.h"
34b321
-#include "qapi/qmp/qdict.h"
34b321
 #include "qemu-common.h"
34b321
 #include "qapi/qmp/json-lexer.h"
34b321
 #include "qapi/qmp/json-streamer.h"
34b321
@@ -34,7 +30,7 @@ static void json_message_process_token(JSONLexer *lexer, GString *input,
34b321
                                        JSONTokenType type, int x, int y)
34b321
 {
34b321
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
34b321
-    QDict *dict;
34b321
+    JSONToken *token;
34b321
 
34b321
     switch (type) {
34b321
     case JSON_LCURLY:
34b321
@@ -53,15 +49,16 @@ static void json_message_process_token(JSONLexer *lexer, GString *input,
34b321
         break;
34b321
     }
34b321
 
34b321
-    dict = qdict_new();
34b321
-    qdict_put(dict, "type", qint_from_int(type));
34b321
-    qdict_put(dict, "token", qstring_from_str(input->str));
34b321
-    qdict_put(dict, "x", qint_from_int(x));
34b321
-    qdict_put(dict, "y", qint_from_int(y));
34b321
+    token = g_malloc(sizeof(JSONToken) + input->len + 1);
34b321
+    token->type = type;
34b321
+    memcpy(token->str, input->str, input->len);
34b321
+    token->str[input->len] = 0;
34b321
+    token->x = x;
34b321
+    token->y = y;
34b321
 
34b321
     parser->token_size += input->len;
34b321
 
34b321
-    g_queue_push_tail(parser->tokens, dict);
34b321
+    g_queue_push_tail(parser->tokens, token);
34b321
 
34b321
     if (type == JSON_ERROR) {
34b321
         goto out_emit_bad;
34b321
-- 
34b321
1.8.3.1
34b321