Merge pull request #307 from MihailRis/update-toml

Update TOML parser

src/coders/commons.cpp

@@ -108,6 +108,10 @@ bool BasicParser::hasNext() {
     return pos < source.length();
 }
 
+size_t BasicParser::remain() const {
+    return source.length() - pos;
+}
+
 bool BasicParser::isNext(const std::string& substring) {
     if (source.length() - pos < substring.length()) {
         return false;
@@ -221,10 +225,6 @@ std::string_view BasicParser::readUntilEOL() {
 std::string BasicParser::parseName() {
     char c = peek();
     if (!is_identifier_start(c)) {
-        if (c == '"') {
-            pos++;
-            return parseString(c);
-        }
         throw error("identifier expected");
     }
     int start = pos;
@@ -234,6 +234,18 @@ std::string BasicParser::parseName() {
     return std::string(source.substr(start, pos - start));
 }
 
+std::string BasicParser::parseXmlName() {
+    char c = peek();
+    if (!is_json_identifier_start(c)) {
+        throw error("identifier expected");
+    }
+    int start = pos;
+    while (hasNext() && is_json_identifier_part(source[pos])) {
+        pos++;
+    }
+    return std::string(source.substr(start, pos - start));
+}
+
 int64_t BasicParser::parseSimpleInt(int base) {
     char c = peek();
     int index = hexchar2int(c);
@@ -349,36 +361,16 @@ std::string BasicParser::parseString(char quote, bool closeRequired) {
                 continue;
             }
             switch (c) {
-                case 'n':
+                case 'n': ss << '\n'; break;
-                    ss << '\n';
+                case 'r': ss << '\r'; break;
-                    break;
+                case 'b': ss << '\b'; break;
-                case 'r':
+                case 't': ss << '\t'; break;
-                    ss << '\r';
+                case 'f': ss << '\f'; break;
-                    break;
+                case '\'': ss << '\\'; break;
-                case 'b':
+                case '"': ss << '"'; break;
-                    ss << '\b';
+                case '\\': ss << '\\'; break;
-                    break;
+                case '/': ss << '/'; break;
-                case 't':
+                case '\n': continue;
-                    ss << '\t';
-                    break;
-                case 'f':
-                    ss << '\f';
-                    break;
-                case '\'':
-                    ss << '\\';
-                    break;
-                case '"':
-                    ss << '"';
-                    break;
-                case '\\':
-                    ss << '\\';
-                    break;
-                case '/':
-                    ss << '/';
-                    break;
-                case '\n':
-                    pos++;
-                    continue;
                 default:
                     throw error(
                         "'\\" + std::string({c}) + "' is an illegal escape"

src/coders/commons.hpp

@@ -30,14 +30,22 @@ inline bool is_whitespace(int c) {
 }
 
 inline bool is_identifier_start(int c) {
-    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' ||
+    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
-           c == '.';
 }
 
 inline bool is_identifier_part(int c) {
     return is_identifier_start(c) || is_digit(c) || c == '-';
 }
 
+inline bool is_json_identifier_start(int c) {
+    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' ||
+           c == '.';
+}
+
+inline bool is_json_identifier_part(int c) {
+    return is_json_identifier_start(c) || is_digit(c) || c == '-';
+}
+
 inline int hexchar2int(int c) {
     if (c >= '0' && c <= '9') {
         return c - '0';
@@ -99,7 +107,9 @@ public:
     std::string_view readUntil(char c);
     std::string_view readUntilEOL();
     std::string parseName();
+    std::string parseXmlName();
     bool hasNext();
+    size_t remain() const;
     char peek();
     char peekInLine();
     char peekNoJump();

src/coders/json.cpp

@@ -178,7 +178,8 @@ dv::value Parser::parseObject() {
             skipLine();
             continue;
         }
-        std::string key = parseName();
+        expect('"');
+        std::string key = parseString('"');
         char next = peek();
         if (next != ':') {
             throw error("':' expected");

src/coders/toml.cpp

@@ -26,33 +26,151 @@ class TomlReader : BasicParser {
         }
     }
 
-    dv::value& getSection(const std::string& section) {
+    // modified version of BaseParser.parseString
-        if (section.empty()) {
+    // todo: extract common part
-            return root;
+    std::string parseMultilineString() {
-        }
+        pos += 2;
-        size_t offset = 0;
+        char next = peek();
-        auto rootMap = &root;
+
-        do {
+        std::stringstream ss;
-            size_t index = section.find('.', offset);
+        while (hasNext()) {
-            if (index == std::string::npos) {
+            char c = source[pos];
-                auto map = rootMap->at(section);
+            if (c == '"' && remain() >= 2 && 
-                if (!map) {
+                source[pos+1] == '"' && 
-                    return rootMap->object(section);
+                source[pos+2] == '"') {
+                pos += 3;
+                return ss.str();
+            }
+            if (c == '\\') {
+                pos++;
+                c = nextChar();
+                if (c >= '0' && c <= '7') {
+                    pos--;
+                    ss << (char)parseSimpleInt(8);
+                    continue;
                 }
-                return *map;
+                switch (c) {
+                    case 'n': ss << '\n'; break;
+                    case 'r': ss << '\r'; break;
+                    case 'b': ss << '\b'; break;
+                    case 't': ss << '\t'; break;
+                    case 'f': ss << '\f'; break;
+                    case '\'': ss << '\\'; break;
+                    case '"': ss << '"'; break;
+                    case '\\': ss << '\\'; break;
+                    case '/': ss << '/'; break;
+                    case '\n': continue;
+                    default:
+                        throw error(
+                            "'\\" + std::string({c}) + "' is an illegal escape"
+                        );
+                }
+                continue;
             }
-            auto subsection = section.substr(offset, index);
+            ss << c;
-            auto map = rootMap->at(subsection);
+            pos++;
-            if (!map) {
+        }
-                rootMap = &rootMap->object(subsection);
+        throw error("unexpected end");
-            } else {
-                rootMap = &*map;
-            }
-            offset = index + 1;
-        } while (true);
     }
 
-    void readSection(const std::string& section, dv::value& map) {
+    dv::value parseValue() {
+        char c = peek();
+        if (is_digit(c)) {
+            int start = pos;
+            // parse numeric literal
+            auto value = parseNumber(1);
+            if (hasNext() && peekNoJump() == '-') {
+                while (hasNext()) {
+                    c = source[pos];
+                    if (!is_digit(c) && c != ':' && c != '.' && c != '-' &&
+                        c != 'T' && c != 'Z') {
+                        break;
+                    }
+                    pos++;
+                }
+                return std::string(source.substr(start, pos - start));
+            }
+            return value;
+        } else if (c == '-' || c == '+') {
+            int sign = c == '-' ? -1 : 1;
+            pos++;
+            // parse numeric literal
+            return parseNumber(sign);
+        } else if (is_identifier_start(c)) {
+            // parse keywords
+            std::string keyword = parseName();
+            if (keyword == "true" || keyword == "false") {
+                return keyword == "true";
+            } else if (keyword == "inf") {
+                return INFINITY;
+            } else if (keyword == "nan") {
+                return NAN;
+            }
+            throw error("unknown keyword " + util::quote(keyword));
+        } else if (c == '"' || c == '\'') {
+            pos++;
+            if (remain() >= 2 && 
+                c  == '"' && 
+                source[pos] == '"' && 
+                source[pos+1] == '"') {
+                return parseMultilineString();
+            }
+            return parseString(c);
+        } else if (c == '[') {
+            // parse array
+            pos++;
+            dv::list_t values;
+            while (peek() != ']') {
+                values.push_back(parseValue());
+                if (peek() != ']') {
+                    expect(',');
+                }
+            }
+            pos++;
+            return dv::value(std::move(values));
+        } else if (c == '{') {
+            // parse inline table
+            pos++;
+            auto table = dv::object();
+            while (peek() != '}') {
+                auto key = parseName();
+                expect('=');
+                table[key] = parseValue();
+                if (peek() != '}') {
+                    expect(',');
+                }
+            }
+            pos++;
+            return table;
+        } else {
+            throw error("feature is not supported");
+        }
+    }
+
+    dv::value& parseLValue(dv::value& root) {
+        dv::value* lvalue = &root;
+        while (hasNext()) {
+            char c = peek();
+            std::string name;
+            if (c == '\'' || c == '"') {
+                pos++;
+                name = parseString(c);
+            } else {
+                name = parseName();
+            }
+            if (lvalue->getType() == dv::value_type::none) {
+                *lvalue = dv::object();
+            }
+            lvalue = &(*lvalue)[name];
+            if (peek() != '.') {
+                break;
+            }
+            pos++;
+        }
+        return *lvalue;
+    }
+
+    void readSection(dv::value& map, dv::value& root) {
         while (hasNext()) {
             skipWhitespace();
             if (!hasNext()) {
@@ -60,43 +178,43 @@ class TomlReader : BasicParser {
             }
             char c = nextChar();
             if (c == '[') {
-                std::string name = parseName();
+                if (hasNext() && peek() == '[') {
-                pos++;
+                    pos++;
-                readSection(name, getSection(name));
+                    // parse list of tables
+                    dv::value& list = parseLValue(root);
+                    if (list == nullptr) {
+                        list = dv::list();
+                    } else if (!list.isList()) {
+                        throw error("target is not an array");
+                    }
+                    expect(']');
+                    expect(']');
+                    dv::value section = dv::object();
+                    readSection(section, root);
+                    list.add(std::move(section));
+                    return;
+                }
+                // parse table
+                dv::value& section = parseLValue(root);
+                if (section == nullptr) {
+                    section = dv::object();
+                } else if (!section.isObject()) {
+                    throw error("target is not a table");
+                }
+                expect(']');
+                readSection(section, root);
                 return;
             }
             pos--;
-            std::string name = parseName();
+            dv::value& lvalue = parseLValue(map);
             expect('=');
-            c = peek();
+            lvalue = parseValue();
-            if (is_digit(c)) {
-                map[name] = parseNumber(1);
-            } else if (c == '-' || c == '+') {
-                int sign = c == '-' ? -1 : 1;
-                pos++;
-                map[name] = parseNumber(sign);
-            } else if (is_identifier_start(c)) {
-                std::string identifier = parseName();
-                if (identifier == "true" || identifier == "false") {
-                    map[name] = identifier == "true";
-                } else if (identifier == "inf") {
-                    map[name] = INFINITY;
-                } else if (identifier == "nan") {
-                    map[name] = NAN;
-                }
-            } else if (c == '"' || c == '\'') {
-                pos++;
-                map[name] = parseString(c);
-            } else {
-                throw error("feature is not supported");
-            }
             expectNewLine();
         }
     }
 public:
     TomlReader(std::string_view file, std::string_view source)
-        : BasicParser(file, source) {
+        : BasicParser(file, source), root(dv::object()) {
-        root = dv::object();
     }
 
     dv::value read() {
@@ -104,7 +222,7 @@ public:
         if (!hasNext()) {
             return std::move(root);
         }
-        readSection("", root);
+        readSection(root, root);
         return std::move(root);
     }
 };
@@ -113,6 +231,7 @@ void toml::parse(
     SettingsHandler& handler, std::string_view file, std::string_view source
 ) {
     auto map = parse(file, source);
+    
     for (const auto& [sectionName, sectionMap] : map.asObject()) {
         if (!sectionMap.isObject()) {
             continue;

src/data/dv.hpp

@@ -196,6 +196,9 @@ namespace dv {
         value(std::shared_ptr<objects::Bytes> v) noexcept {
             this->operator=(std::move(v));
         }
+        value(list_t values) {
+            this->operator=(std::make_shared<list_t>(std::move(values)));
+        }
 
         value(const value& v) noexcept : type(value_type::none) {
             this->operator=(v);

src/engine.cpp

@@ -9,6 +9,7 @@
 #include "coders/imageio.hpp"
 #include "coders/json.hpp"
 #include "coders/toml.hpp"
+#include "coders/commons.hpp"
 #include "content/Content.hpp"
 #include "content/ContentBuilder.hpp"
 #include "content/ContentLoader.hpp"
@@ -125,7 +126,12 @@ void Engine::loadSettings() {
     if (fs::is_regular_file(settings_file)) {
         logger.info() << "loading settings";
         std::string text = files::read_string(settings_file);
-        toml::parse(settingsHandler, settings_file.string(), text);
+        try {
+            toml::parse(settingsHandler, settings_file.string(), text);
+        } catch (const parsing_error& err) {
+            logger.error() << err.errorLog();
+            throw;
+        }
     }
 }
 

src/window/Events.cpp

@@ -196,22 +196,25 @@ void Events::loadBindings(
     const std::string& filename, const std::string& source
 ) {
     auto map = toml::parse(filename, source);
-    for (auto& [key, value] : map.asObject()) {
+    for (auto& [sectionName, section] : map.asObject()) {
-        auto [prefix, codename] = util::split_at(value.asString(), ':');
+        for (auto& [name, value] : section.asObject()) {
-        inputtype type;
+            auto key = sectionName + "." + name;
-        int code;
+            auto [prefix, codename] = util::split_at(value.asString(), ':');
-        if (prefix == "key") {
+            inputtype type;
-            type = inputtype::keyboard;
+            int code;
-            code = static_cast<int>(input_util::keycode_from(codename));
+            if (prefix == "key") {
-        } else if (prefix == "mouse") {
+                type = inputtype::keyboard;
-            type = inputtype::mouse;
+                code = static_cast<int>(input_util::keycode_from(codename));
-            code = static_cast<int>(input_util::mousecode_from(codename));
+            } else if (prefix == "mouse") {
-        } else {
+                type = inputtype::mouse;
-            logger.error()
+                code = static_cast<int>(input_util::mousecode_from(codename));
-                << "unknown input type: " << prefix << " (binding "
+            } else {
-                << util::quote(key) << ")";
+                logger.error()
-            continue;
+                    << "unknown input type: " << prefix << " (binding "
+                    << util::quote(key) << ")";
+                continue;
+            }
+            Events::bind(key, type, code);
         }
-        Events::bind(key, type, code);
     }
 }

test/coders/toml.cpp

@@ -0,0 +1,95 @@
+#include "coders/toml.hpp"
+
+#include <gtest/gtest.h>
+
+#include "data/dv.hpp"
+#include "util/stringutil.hpp"
+#include "util/Buffer.hpp"
+#include "coders/commons.hpp"
+
+TEST(TOML, EncodeDecode) {
+    const std::string name = "TOML-encoder";
+    const int bytesSize = 20;
+    const int year = 2019;
+    const float score = 3.141592;
+    const bool visible = true;
+    dv::objects::Bytes srcBytes(bytesSize);
+    for (int i = 0; i < bytesSize; i ++) {
+        srcBytes[i] = rand();
+    }
+
+    std::string text;
+    {
+        auto object = dv::object();
+        object["name"] = name;
+        object["year"] = year;
+        object["score"] = score;
+        object["visible"] = visible;
+        object["data"] = srcBytes;
+
+        text = toml::stringify(object, "");
+        std::cout << text << std::endl;
+    }
+    try {
+        auto object = toml::parse("<string>", text);
+        EXPECT_EQ(object["name"].asString(), name);
+        EXPECT_EQ(object["year"].asInteger(), year);
+        EXPECT_FLOAT_EQ(object["score"].asNumber(), score);
+        EXPECT_EQ(object["visible"].asBoolean(), visible);
+        auto b64string = object["data"].asString();
+
+        auto bytes = util::base64_decode(b64string);
+        EXPECT_EQ(bytes.size(), bytesSize);
+        for (int i = 0; i < bytesSize; i++) {
+            EXPECT_EQ(bytes[i], srcBytes[i]);
+        }
+    } catch (const parsing_error& err) {
+        std::cerr << err.errorLog() << std::endl;
+        throw;
+    }
+}
+
+// Modified example from toml.io
+inline std::string SRC_EXAMPLE =
+    "# This is a TOML document\n"
+    "\n"
+    "title = \"TOML Example\"\n"
+    "\n"
+    "[owner]\n"
+    "name = \"Tom Preston-Werner\"\n"
+    "dob = 1979-05-27T07:32:00-08:00\n"
+    "\n"
+    "[database]\n"
+    "enabled = true\n"
+    "ports = [ 8000, 8001, 8002 ]\n"
+    "data = [ [\"delta\", \"phi\"], [3.14] ]\n"
+    "temp_targets = { cpu = 79.5, case = 72.0 }\n"
+    "\n"
+    "[servers]\n"
+    "\n"
+    "[servers.alpha]\n"
+    "ip = \"10.0.0.1\"\n"
+    "role = \"frontend\"\n"
+    "\n"
+    "[servers.beta]\n"
+    "ip = \"10.0.0.2\"\n"
+    "role = \"\"\"back\\\n"
+    "end\"\"\"\n"
+    "\n"
+    "[[users]]\n"
+    "name = \"noname\"\n"
+    "\n"
+    "[[users]]\n"
+    "name = \"user1\"\n"
+    "suspended = true\n";
+
+TEST(TOML, ExampleCode) {
+    try {
+        std::cout << SRC_EXAMPLE << std::endl;
+        auto object = toml::parse("<string>", SRC_EXAMPLE);
+        std::cout << object << std::endl;
+    } catch (const parsing_error& err) {
+        std::cerr << err.errorLog() << std::endl;
+        throw;
+    }
+}