From 8711fc10945d0e93c65627ed5056c6240f8c0e37 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 19:35:00 +0100 Subject: [PATCH] Implement partial parsing support. This commit implements a consume method that takes a chunk of json, append it to the current data and parses it. --- json11.cpp | 271 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 158 insertions(+), 113 deletions(-) diff --git a/json11.cpp b/json11.cpp index b2e00ed..03b4364 100644 --- a/json11.cpp +++ b/json11.cpp @@ -815,40 +815,71 @@ struct JsonParserPriv final { void parse_object() { assert(states.top() >= VALUE_OBJECT && states.top() <= OBJECT_VALUE); - values.push(map()); + switch (states.top()) { + case VALUE_OBJECT: + values.push(map()); + set_state(OBJECT_KEY_OR_END); + break; - set_state(OBJECT_KEY_OR_END); - char ch = get_next_token(); - if (ch == '}') { - pop_state(); - return; - } + case OBJECT_KEY_OR_END: { + char ch = get_next_token(); - while (1) { if (need_data) - return; + break; + + if (ch == '}') + return pop_state(); if (ch != '"') { values.pop(); - return fail("expected '\"' in object, got " + esc(ch)); + fail("expected '\"' in object, got " + esc(ch)); + return; } - set_state(OBJECT_KEY); + set_state(OBJECT_COLON); push_state(VALUE_STRING); - parse_string(); + + break; + } + case OBJECT_COMMA_OR_END: { + char ch = get_next_token(); + if (need_data) + break; + + if (ch == '}') { + pop_state(); return; + } - string key = values.top().string_value(); - values.pop(); + if (ch != ',') { + values.pop(); + fail("expected ',' or '}' in object, got " + esc(ch)); + return; + } + + set_state(OBJECT_KEY); - if (failed) { + break; + } + case OBJECT_KEY: { + char ch = get_next_token(); + + if (need_data) + break; + + if (ch != '"') { values.pop(); - return values.push(Json()); + return fail("expected '\"' in object, got " + esc(ch)); } set_state(OBJECT_COLON); - ch = get_next_token(); + push_state(VALUE_STRING); + + break; + } + case OBJECT_COLON: { + char ch = get_next_token(); if (need_data) return; @@ -859,38 +890,23 @@ struct JsonParserPriv final { set_state(OBJECT_VALUE); push_state(EXPECT_VALUE); - parse_json(); - if (need_data) - return; - + break; + } + case OBJECT_VALUE: { Json value = values.top(); values.pop(); - - if (failed) { - values.pop(); - return values.push(Json()); - } - + string key = values.top().string_value(); + values.pop(); map data = values.top().object_items(); data[std::move(key)] = value; values.pop(); values.push(data); set_state(OBJECT_COMMA_OR_END); - ch = get_next_token(); - if (need_data) - return; - - if (ch == '}') { - pop_state(); - break; - } - - if (ch != ',') { - values.pop(); - return fail("expected ',' in object, got " + esc(ch)); - } - ch = get_next_token(); + break; + } + default: + assert(false); } } @@ -901,59 +917,62 @@ struct JsonParserPriv final { void parse_array() { assert(states.top() >= VALUE_ARRAY && states.top() <= ARRAY_VALUE); - values.push(vector()); + switch (states.top()) { + case VALUE_ARRAY: + values.push(vector()); + set_state(ARRAY_VALUE_OR_END); + break; - set_state(ARRAY_VALUE_OR_END); - char ch = get_next_token(); + case ARRAY_VALUE_OR_END: { + char ch = get_next_token(); - if (ch == ']') { - pop_state(); - return; - } - - while (1) { if (need_data) return; + if (ch == ']') + return pop_state(); + i--; set_state(ARRAY_VALUE); push_state(EXPECT_VALUE); - parse_json(); + break; + } + case ARRAY_COMMA_OR_END: { + char ch = get_next_token(); if (need_data) return; - Json value = values.top(); - values.pop(); - - if (failed) { + if (ch == ']') { + pop_state(); + return; + } + if (ch != ',') { values.pop(); - return values.push(Json()); + fail("expected ',' in list, got " + esc(ch)); + return; } + set_state(ARRAY_VALUE_OR_END); + + break; + } + case ARRAY_VALUE: { + Json value = values.top(); + values.pop(); + vector data = values.top().array_items(); data.push_back(value); values.pop(); values.push(data); set_state(ARRAY_COMMA_OR_END); - ch = get_next_token(); - if (need_data) - return; - - if (ch == ']') { - pop_state(); - break; - } - - if (ch != ',') { - values.pop(); - return fail("expected ',' in list, got " + esc(ch)); - } - ch = get_next_token(); - (void)ch; + break; + } + default: + assert(false); } } @@ -974,42 +993,79 @@ struct JsonParserPriv final { if (ch == '-' || (ch >= '0' && ch <= '9')) { i--; - set_state(VALUE_NUMBER); - return parse_number(); + return set_state(VALUE_NUMBER); } - if (ch == 't') { - set_state(VALUE_TRUE); - return parse_true(); - } + if (ch == 't') + return set_state(VALUE_TRUE); - if (ch == 'f') { - set_state(VALUE_FALSE); - return parse_false(); - } + if (ch == 'f') + return set_state(VALUE_FALSE); - if (ch == 'n') { - set_state(VALUE_NULL); - return parse_null(); - } + if (ch == 'n') + return set_state(VALUE_NULL); - if (ch == '"') { - set_state(VALUE_STRING); - return parse_string(); - } + if (ch == '"') + return set_state(VALUE_STRING); - if (ch == '{') { - set_state(VALUE_OBJECT); - return parse_object(); - } + if (ch == '{') + return set_state(VALUE_OBJECT); - if (ch == '[') { - set_state(VALUE_ARRAY); - return parse_array(); - } + if (ch == '[') + return set_state(VALUE_ARRAY); return fail("expected value, got " + esc(ch)); } + + void consume(const std::string &in = std::string()) { + need_data = false; + str += in; + + /* try to parse as much as possible */ + while (!states.empty()) { + switch (states.top()) { + case EXPECT_VALUE: + parse_json(); + break; + case VALUE_OBJECT: + case OBJECT_KEY_OR_END: + case OBJECT_COMMA_OR_END: + case OBJECT_KEY: + case OBJECT_COLON: + case OBJECT_VALUE: + parse_object(); + break; + case VALUE_ARRAY: + case ARRAY_VALUE_OR_END: + case ARRAY_COMMA_OR_END: + case ARRAY_VALUE: + parse_array(); + break; + case VALUE_STRING: + parse_string(); + break; + case VALUE_NUMBER: + parse_number(); + break; + case VALUE_TRUE: + parse_true(); + break; + case VALUE_FALSE: + parse_false(); + break; + case VALUE_NULL: + parse_null(); + break; + case VALUE_COMMENT: + consume_comment(); + break; + } + + if (failed || need_data) + break; + } + } + }; JsonParser::JsonParser(): @@ -1024,21 +1080,20 @@ JsonParser::~JsonParser() { } void JsonParser::consume(const std::string &in) { - parser->str = in; - parser->eof = true; - parser->parse_json(); + parser->consume(in); } Json JsonParser::json() const { + parser->eof = true; + parser->consume(); return parser->values.top(); } Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; assert(parser.states.size() == 1); - parser.eof = true; - parser.parse_json(); + parser.consume(); // Check for any trailing garbage parser.consume_garbage(); @@ -1047,12 +1102,6 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { return Json(); } - if (parser.need_data) { - /* when doing full parsing, this is an error */ - parser.failed = true; - parser.values.push(Json()); - } - #ifndef NDEBUG if (!parser.failed) { assert(parser.values.size() == 1); @@ -1072,11 +1121,7 @@ vector Json::parse_multi(const string &in, parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed && !parser.need_data) { - parser.parse_json(); - if (parser.need_data) { - parser.failed = true; - parser.values.push(Json()); - } + parser.consume(); #ifndef NDEBUG if (!parser.failed) assert(parser.values.size() == 1);