Merge pull request #493 from MihailRis/yaml

yaml

doc/en/scripting.md

@@ -11,7 +11,7 @@ Subsections:
 - [Libraries](#)
     - [app](scripting/builtins/libapp.md)
     - [base64](scripting/builtins/libbase64.md)
-    - [bjson, json, toml](scripting/filesystem.md)
+    - [bjson, json, toml, yaml](scripting/filesystem.md)
     - [block](scripting/builtins/libblock.md)
     - [byteutil](scripting/builtins/libbyteutil.md)
     - [cameras](scripting/builtins/libcameras.md)

doc/en/scripting/filesystem.md

@@ -36,6 +36,22 @@ toml.parse(code: str) -> table
 
 Parses a TOML string into a table.
 
+## *yaml* library
+
+The library contains functions for serializing and deserializing tables:
+
+```python
+yaml.tostring(object: table) -> str
+```
+
+Serializes an object into a YAML string.
+
+```python
+yaml.parse(code: str) -> table
+```
+
+Parses a YAML string into a table.
+
 ## *bjson* library
 
 The library contains functions for working with the binary data exchange format [vcbjson](../../specs/binary_json_spec.md).

doc/ru/scripting.md

@@ -11,7 +11,7 @@
 - [Библиотеки](#)
     - [app](scripting/builtins/libapp.md)
     - [base64](scripting/builtins/libbase64.md)
-    - [bjson, json, toml](scripting/filesystem.md)
+    - [bjson, json, toml, yaml](scripting/filesystem.md)
     - [block](scripting/builtins/libblock.md)
     - [byteutil](scripting/builtins/libbyteutil.md)
     - [cameras](scripting/builtins/libcameras.md)

doc/ru/scripting/filesystem.md

@@ -36,6 +36,22 @@ toml.parse(code: str) -> table
 
 Парсит TOML строку в таблицу.
 
+## Библиотека yaml
+
+Библиотека содержит функции для сериализации и десериализации таблиц:
+
+```python
+yaml.tostring(object: table) -> str
+```
+
+Сериализует объект в YAML строку.
+
+```python
+yaml.parse(code: str) -> table
+```
+
+Парсит YAML строку в таблицу.
+
 ## Библиотека bjson
 
 Библиотека содержит функции для работы с двоичным форматом обмена данными [vcbjson](../../specs/binary_json_spec.md).

src/coders/BasicParser.hpp

@@ -7,16 +7,20 @@ template <typename CharT>
 class BasicParser {
     using StringT = std::basic_string<CharT>;
     using StringViewT = std::basic_string_view<CharT>;
+
+    void skipWhitespaceHashComment(bool newline = true);
 protected:
     std::string_view filename;
     StringViewT source;
     uint pos = 0;
     uint line = 1;
     uint linestart = 0;
+    bool hashComment = false;
 
-    virtual void skipWhitespace();
+    void skipWhitespace(bool newline = true);
     void skip(size_t n);
     void skipLine();
+    void skipEmptyLines();
     bool skipTo(const StringT& substring);
     void expect(CharT expected);
     void expect(const StringT& substring);

src/coders/BasicParser.inl

@@ -31,10 +31,17 @@ namespace {
 }
 
 template<typename CharT>
-void BasicParser<CharT>::skipWhitespace() {
+void BasicParser<CharT>::skipWhitespace(bool newline) {
+    if (hashComment) {
+        skipWhitespaceHashComment(newline);
+        return;
+    }
     while (hasNext()) {
         char next = source[pos];
         if (next == '\n') {
+            if (!newline) {
+                break;
+            }
             line++;
             linestart = ++pos;
             continue;
@@ -47,6 +54,36 @@ void BasicParser<CharT>::skipWhitespace() {
     }
 }
 
+template<typename CharT>
+void BasicParser<CharT>::skipWhitespaceHashComment(bool newline) {
+    while (hasNext()) {
+        char next = source[pos];
+        if (next == '\n') {
+            if (!newline) {
+                break;
+            }
+            line++;
+            linestart = ++pos;
+            continue;
+        }
+        if (is_whitespace(next)) {
+            pos++;
+        } else {
+            break;
+        }
+    }
+    if (hasNext() && source[pos] == '#') {
+        if (!newline) {
+            readUntilEOL();
+            return;
+        }
+        skipLine();
+        if (hasNext() && (is_whitespace(source[pos]) || source[pos] == '#')) {
+            skipWhitespaceHashComment(newline);
+        }
+    }
+}
+
 template<typename CharT>
 void BasicParser<CharT>::skip(size_t n) {
     n = std::min(n, source.length() - pos);
@@ -73,6 +110,12 @@ void BasicParser<CharT>::skipLine() {
     }
 }
 
+template<typename CharT>
+void BasicParser<CharT>::skipEmptyLines() {
+    skipWhitespace();
+    pos = linestart;
+}
+
 template<typename CharT>
 bool BasicParser<CharT>::skipTo(const std::basic_string<CharT>& substring) {
     size_t idx = source.find(substring, pos);
@@ -240,9 +283,12 @@ std::basic_string_view<CharT> BasicParser<CharT>::readUntilWhitespace() {
 template <typename CharT>
 std::basic_string_view<CharT> BasicParser<CharT>::readUntilEOL() {
     int start = pos;
-    while (hasNext() && source[pos] != '\r' && source[pos] != '\n') {
+    while (hasNext() && source[pos] != '\n') {
         pos++;
     }
+    if (pos > start && source[pos - 1] == '\r') {
+        return source.substr(start, pos - start - 1);
+    }
     return source.substr(start, pos - start);
 }
 

src/coders/json.cpp

@@ -13,13 +13,14 @@ using namespace json;
 
 namespace {
     class Parser : BasicParser<char> {
-        dv::value parseList();
-        dv::value parseObject();
-        dv::value parseValue();
-    public:
+        public:
         Parser(std::string_view filename, std::string_view source);
 
         dv::value parse();
+    private:
+        dv::value parseList();
+        dv::value parseObject();
+        dv::value parseValue();
     };
 }
 

src/coders/toml.cpp

@@ -16,16 +16,6 @@ using namespace toml;
 class TomlReader : BasicParser<char> {
     dv::value root;
 
-    void skipWhitespace() override {
-        BasicParser::skipWhitespace();
-        if (hasNext() && source[pos] == '#') {
-            skipLine();
-            if (hasNext() && is_whitespace(peek())) {
-                skipWhitespace();
-            }
-        }
-    }
-
     // modified version of BaseParser.parseString
     // todo: extract common part
     std::string parseMultilineString() {
@@ -214,6 +204,7 @@ class TomlReader : BasicParser<char> {
 public:
     TomlReader(std::string_view file, std::string_view source)
         : BasicParser(file, source), root(dv::object()) {
+        hashComment = true;
     }
 
     dv::value read() {

src/coders/yaml.cpp

@@ -0,0 +1,459 @@
+#include "yaml.hpp"
+#include "BasicParser.hpp"
+
+#include <iomanip>
+
+using namespace yaml;
+
+namespace {
+    enum Chomping {
+        CLIP, STRIP, KEEP
+    };
+
+    class Parser : BasicParser<char> {
+    public:
+        Parser(std::string_view filename, std::string_view source);
+
+        dv::value parseValue();
+        dv::value parseFullValue(int indent);
+        dv::value parseArray(int indent = 0);
+        dv::value parseObject(dv::value&& object, int indent = 0);
+        dv::value parseInlineArray();
+        dv::value parseInlineObject();
+    private:
+        int countIndent();
+        bool expectIndent(int indent);
+        std::string_view readYamlIdentifier();
+        std::string readMultilineString(int indent, bool eols, Chomping chomp);
+    };
+}
+
+inline bool is_yaml_identifier_char(int c) {
+    return c > 20 && c != ':' && c != ' ' && c != '\n' && c != '\r' &&
+           c != '\t' && c != '\f' && c != '\v';
+}
+
+static dv::value perform_literal(std::string_view literal) {
+    if (literal == "true" || literal == "True" || 
+        literal == "false" || literal == "False") {
+        return literal[0] == 't';
+    }
+    if (literal == "null" || literal == "Null") {
+        return nullptr;
+    }
+    return std::string(literal);
+}
+
+Parser::Parser(std::string_view filename, std::string_view source)
+    : BasicParser(filename, source) {
+    hashComment = true;
+}
+
+bool Parser::expectIndent(int required) {
+    int indent = 0;
+    while (hasNext() && source[pos] == ' ' && indent < required) {
+        indent++;
+        pos++;
+    }
+    return indent >= required;
+}
+
+std::string Parser::readMultilineString(int indent, bool eols, Chomping chomp) {
+    int next_indent = countIndent();
+    if (next_indent <= indent) {
+        throw error("indentation error");
+    }
+    std::stringstream ss;
+    ss << readUntilEOL();
+    if (hasNext()) {
+        skip(1);
+    }
+    int trailingEmpties = 0;
+    while (true) {
+        while (expectIndent(next_indent)) {
+            trailingEmpties = 0;
+            ss << (eols ? '\n' : ' ');
+            ss << readUntilEOL();
+            if (hasNext()) {
+                skip(1);
+            }
+        }
+        while (true) {
+            skipWhitespace(false);
+            if (!hasNext() || source[pos] != '\n') {
+                break;
+            }
+            skip(1);
+            trailingEmpties++;
+        }
+        if (!expectIndent(next_indent)) {
+            break;
+        }
+        pos = linestart;
+    }
+    if (chomp == KEEP) {
+        for (int i = 0; i < trailingEmpties - 1; i++) {
+            ss << (eols ? '\n' : ' ');
+        }
+    }
+    ss << '\n';
+
+    pos = linestart;
+
+    auto string = ss.str();
+    if (chomp == STRIP) {
+        util::trim(string);
+    }
+    return string;
+}
+
+std::string_view Parser::readYamlIdentifier() {
+    char c = peek();
+    if (!is_yaml_identifier_char(c)) {
+        throw error("identifier expected");
+    }
+    int start = pos;
+    while (hasNext() && is_yaml_identifier_char(source[pos])) {
+        pos++;
+    }
+    return source.substr(start, pos - start);
+}
+
+int Parser::countIndent() {
+    int indent = 0;
+    while (hasNext() && source[pos] == ' ') {
+        indent++;
+        pos++;
+    }
+    return indent;
+}
+
+dv::value Parser::parseValue() {
+    char c = peek();
+    if (is_digit(c)) {
+        return parseNumber(1);
+    } else if (c == '-' || c == '+') {
+        skip(1);
+        return parseNumber(c == '-' ? -1 : 1);
+    } else if (c == '"' || c == '\'') {
+        skip(1);
+        return parseString(c, true);
+    } else if (c == '[') {
+        return parseInlineArray();
+    } else if (c == '{') {
+        return parseInlineObject();
+    } else {
+        return perform_literal(readUntilEOL());
+    }
+    throw error("unexpected character");
+}
+
+dv::value Parser::parseInlineArray() {
+    expect('[');
+    auto list = dv::list();
+    while (peek() != ']') {
+        if (peek() == '#') {
+            skipLine();
+            continue;
+        }
+        list.add(parseValue());
+
+        char next = peek();
+        if (next == ',') {
+            pos++;
+        } else if (next == ']') {
+            break;
+        } else {
+            throw error("',' expected");
+        }
+    }
+    pos++;
+    return list;
+}
+
+dv::value Parser::parseInlineObject() {
+    expect('{');
+    dv::value object = dv::object();
+    while (peek() != '}') {
+        if (peek() == '#') {
+            skipLine();
+            continue;
+        }
+        auto name = readYamlIdentifier();
+        expect(':');
+        object[std::string(name)] = parseValue();
+
+        char next = peek();
+        if (next == ',') {
+            pos++;
+        } else if (next == '}') {
+            break;
+        } else {
+            throw error("',' expected");
+        }
+    }
+    pos++;
+    return object;
+}
+
+dv::value Parser::parseFullValue(int indent) {
+    dv::value value;
+    char c = source[pos];
+    if (c == '\n') {
+        skip(1);
+        skipEmptyLines();
+        int init_pos = pos;
+        int next_indent = countIndent();
+        if (next_indent < indent) {
+            throw error("indentation error");
+        }
+        if (source[pos] == '-') {
+            pos = init_pos;
+            return parseArray(next_indent);
+        } else {
+            pos = init_pos;
+            return parseObject(dv::object(), next_indent);
+        }
+    } else if (is_digit(c)) {
+        return parseNumber(1);
+    } else if (c == '-' || c == '+') {
+        skip(1);
+        return parseNumber(c == '-' ? -1 : 1);
+    } else if (c == '"' || c == '\'') {
+        skip(1);
+        return parseString(c, true);
+    } else if (c == '[') {
+        return parseInlineArray();
+    } else if (c == '{') {
+        return parseInlineObject();
+    } else if (c == '|' || c == '>') {
+        skip(1);
+        Chomping chomp = CLIP;
+        if (source[pos] == '-' || source[pos] == '+') {
+            chomp = source[pos] == '-' ? STRIP : KEEP;
+            skip(1);
+        }
+        skipWhitespace(false);
+        expectNewLine();
+        return readMultilineString(indent, c == '|', chomp);
+    } else {
+        return perform_literal(readUntilEOL());
+    }
+}
+
+dv::value Parser::parseArray(int indent) {
+    dv::value list = dv::list();
+
+    while (hasNext()) {
+        skipEmptyLines();
+        int next_indent = countIndent();
+        if (next_indent < indent) {
+            pos = linestart;
+            break;
+        }
+        expect('-');
+        skipWhitespace();
+        size_t nlpos = source.find('\n', pos);
+        size_t colonpos = source.find(':', pos);
+        if (nlpos == std::string::npos && colonpos == std::string::npos) {
+            list.add(perform_literal(readUntilEOL()));
+            break;
+        }
+        if (nlpos < colonpos) {
+            list.add(parseFullValue(next_indent));
+            skipLine();
+        } else {
+            auto name = readYamlIdentifier();
+            expect(':');
+            skipWhitespace(false);
+            dv::value object = dv::object();
+            object[std::string(name)] = parseFullValue(next_indent);
+            skipEmptyLines();
+            next_indent = countIndent();
+            if (next_indent > indent) {
+                pos = linestart;
+                object = parseObject(std::move(object), next_indent);
+            } else {
+                pos = linestart;
+            }
+            list.add(std::move(object));
+        }
+    }
+    return list;
+}
+
+dv::value Parser::parseObject(dv::value&& object, int indent) {
+    skipEmptyLines();
+    while (hasNext()) {
+        size_t prev_pos = pos;
+        int next_indent = countIndent();
+        if (source[pos] == '\n') {
+            skip(1);
+            continue;
+        }
+        if (next_indent < indent) {
+            pos = prev_pos;
+            break;
+        }
+        char c = peek();
+        if (!is_yaml_identifier_char(c)) {
+            if (!is_whitespace(c)) {
+                throw error("invalid character");
+            }
+            continue;
+        }
+        auto name = readYamlIdentifier();
+        expect(':');
+        skipWhitespace(false);
+        object[std::string(name)] = parseFullValue(indent);
+        skipEmptyLines();
+    }
+    return object;
+}
+
+dv::value yaml::parse(std::string_view filename, std::string_view source) {
+    return Parser(filename, source).parseObject(dv::object());
+}
+
+dv::value yaml::parse(std::string_view source) {
+    return parse("[string]", source);
+}
+
+static void add_indent(std::stringstream& ss, int indent) {
+    for (int i = 0; i < indent; i++) {
+        ss << "  ";
+    }
+}
+
+static void insert_string(
+    std::stringstream& ss, const std::string& string, int indent
+) {
+    bool has_spec_chars = false;
+    bool multiline = false;
+    for (char c : string) {
+        if (c < ' ' || c == '"' || c == '\'') {
+            has_spec_chars = true;
+            if (multiline) {
+                break;
+            }
+        }
+        if (c == '\n') {
+            multiline = true;
+            break;
+        }
+    }
+    if (multiline) {
+        ss << "|-\n";
+        size_t offset = 0;
+        size_t newoffset = 0;
+
+        do {
+            offset = newoffset;
+            if (offset == string.length() - 1 && string[offset] == '\n') {
+                break;
+            }
+            add_indent(ss, indent);
+            newoffset = string.find('\n', offset + 1);
+            if (newoffset == std::string::npos) {
+                ss << string.substr(offset);
+                break;
+            } else {
+                ss << string.substr(offset + 1, newoffset - offset - 1);
+            }
+            ss << '\n';
+        } while (true);
+    } else {
+        if (has_spec_chars || string.empty()) {
+            ss << util::escape(string, false);
+        } else {
+            ss << string;
+        }
+    }
+}
+
+static void to_string(
+    std::stringstream& ss,
+    const dv::value& value,
+    int indent,
+    bool eliminateIndent = false
+) {
+    using dv::value_type;
+
+    switch (value.getType()) {
+        case value_type::string:
+            insert_string(ss, value.asString(), indent);
+            break;
+        case value_type::number:
+            ss << std::setprecision(15) << value.asNumber();
+            break;
+        case value_type::integer:
+            ss << value.asInteger();
+            break;
+        case value_type::boolean:
+            ss << (value.asBoolean() ? "true" : "false");
+            break;
+        case value_type::none:
+            ss << "null";
+            break;
+        case value_type::object: {
+            if (value.empty()) {
+                ss << "{}";
+                break;
+            }
+            bool first = true;
+            for (const auto& [key, elem] : value.asObject()) {
+                if (!first) {
+                    ss << '\n';   
+                }
+                if (!eliminateIndent) {
+                    add_indent(ss, indent);
+                } else {
+                    eliminateIndent = false;
+                }
+                ss << key << ": ";
+                if ((elem.isObject() || elem.isList()) && !elem.empty()) {
+                    ss << "\n";
+                    to_string(ss, elem, indent + 1);
+                } else {
+                    to_string(ss, elem, indent + 1);
+                }
+                first = false;
+            }
+            break;
+        }
+        case value_type::list: {
+            if (value.empty()) {
+                ss << "[]";
+                break;
+            }
+            bool first = true;
+            for (const auto& elem : value) {
+                if (!first) {
+                    ss << '\n';   
+                }
+                if (!eliminateIndent) {
+                    add_indent(ss, indent);
+                } else {
+                    eliminateIndent = false;
+                }
+                ss << "- ";
+                to_string(ss, elem, indent + 1, true);
+                first = false;
+            }
+            break;
+        }
+        case value_type::bytes: {
+            const auto& bytes = value.asBytes();
+            auto b64 = util::base64_encode(bytes.data(), bytes.size());
+            b64 = util::join(util::split_by_n(b64, 64), '\n');
+            insert_string(ss, b64, indent);
+            break;
+        }
+    }
+}
+
+std::string yaml::stringify(const dv::value& value) {
+    std::stringstream ss;
+    to_string(ss, value, 0);
+    return ss.str();
+}

src/coders/yaml.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+
+#include "data/dv.hpp"
+
+namespace yaml {
+    dv::value parse(std::string_view filename, std::string_view source);
+    dv::value parse(std::string_view source);
+
+    std::string stringify(const dv::value& value);
+}

src/frontend/locale.cpp

@@ -40,18 +40,10 @@ const std::string& langs::Lang::getId() const {
 /// @brief Language key-value txt files parser
 namespace {
     class Reader : BasicParser<char> {
-        void skipWhitespace() override {
-            BasicParser::skipWhitespace();
-            if (hasNext() && source[pos] == '#') {
-                skipLine();
-                if (hasNext() && is_whitespace(peek())) {
-                    skipWhitespace();
-                }
-            }
-        }
     public:
         Reader(std::string_view file, std::string_view source)
             : BasicParser(file, source) {
+            hashComment = true;
         }
 
         void read(langs::Lang& lang, const std::string &prefix) {

src/logic/scripting/lua/libs/api_lua.hpp

@@ -14,6 +14,7 @@
 ///    int l_function_name(lua::State* L);
 
 // Libraries
+extern const luaL_Reg applib[];
 extern const luaL_Reg audiolib[];
 extern const luaL_Reg base64lib[];
 extern const luaL_Reg bjsonlib[];
@@ -38,7 +39,6 @@ extern const luaL_Reg packlib[];
 extern const luaL_Reg particleslib[]; // gfx.particles
 extern const luaL_Reg playerlib[];
 extern const luaL_Reg quatlib[];
-extern const luaL_Reg applib[];
 extern const luaL_Reg text3dlib[]; // gfx.text3d
 extern const luaL_Reg timelib[];
 extern const luaL_Reg tomllib[];
@@ -48,6 +48,7 @@ extern const luaL_Reg vec3lib[];  // vecn.cpp
 extern const luaL_Reg vec4lib[];  // vecn.cpp
 extern const luaL_Reg weatherlib[];
 extern const luaL_Reg worldlib[];
+extern const luaL_Reg yamllib[];
 
 // Components
 extern const luaL_Reg skeletonlib[];

src/logic/scripting/lua/libs/libjson.cpp

@@ -19,4 +19,5 @@ static int l_json_parse(lua::State* L) {
 const luaL_Reg jsonlib[] = {
     {"tostring", lua::wrap<l_json_stringify>},
     {"parse", lua::wrap<l_json_parse>},
-    {NULL, NULL}};
+    {NULL, NULL}
+};

src/logic/scripting/lua/libs/libyaml.cpp

@@ -0,0 +1,20 @@
+#include "coders/yaml.hpp"
+#include "api_lua.hpp"
+
+static int l_stringify(lua::State* L) {
+    auto value = lua::tovalue(L, 1);
+    auto string = yaml::stringify(value);
+    return lua::pushstring(L, string);
+}
+
+static int l_parse(lua::State* L) {
+    auto string = lua::require_string(L, 1);
+    auto element = yaml::parse("[string]", string);
+    return lua::pushvalue(L, element);
+}
+
+const luaL_Reg yamllib[] = {
+    {"tostring", lua::wrap<l_stringify>},
+    {"parse", lua::wrap<l_parse>},
+    {NULL, NULL}
+};

src/logic/scripting/lua/lua_engine.cpp

@@ -56,6 +56,7 @@ static void create_libs(State* L, StateType stateType) {
     openlib(L, "vec2", vec2lib);
     openlib(L, "vec3", vec3lib);
     openlib(L, "vec4", vec4lib);
+    openlib(L, "yaml", yamllib);
 
     if (stateType == StateType::SCRIPT) {
         openlib(L, "app", applib);

src/util/stringutil.hpp

@@ -115,4 +115,29 @@ namespace util {
     std::string format_data_size(size_t size);
 
     std::pair<std::string, std::string> split_at(std::string_view view, char c);
+
+    template <typename CharT>
+    std::vector<std::basic_string<CharT>> split_by_n(
+        const std::basic_string<CharT>& str, size_t n
+    ) {
+        std::vector<std::basic_string<CharT>> result;
+        for (size_t i = 0; i < str.length(); i += n) {
+            result.push_back(str.substr(i, n));
+        }
+        return result;
+    }
+
+    template <typename CharT>
+    std::basic_string<CharT> join(
+        const std::vector<std::basic_string<CharT>>& strings, CharT delimiter
+    ) {
+        std::basic_stringstream<CharT> ss;
+        for (size_t i = 0; i < strings.size(); ++i) {
+            ss << strings[i];
+            if (i != strings.size() - 1) {
+                ss << delimiter;
+            }
+        }
+        return ss.str();
+    }
 }

test/coders/yaml.cpp

@@ -0,0 +1,22 @@
+#include <gtest/gtest.h>
+
+#include "coders/yaml.hpp"
+#include "coders/json.hpp"
+#include "coders/commons.hpp"
+
+#include "io/io.hpp"
+#include "io/devices/StdfsDevice.hpp"
+
+namespace fs = std::filesystem;
+
+TEST(YAML, EncodeDecode) {
+    io::set_device("root", std::make_shared<io::StdfsDevice>(fs::u8path("../../")));
+    auto filename = "root:.github/workflows/windows-clang.yml";
+    try {
+        auto value = yaml::parse(io::read_string(filename));
+        std::cout << yaml::stringify(value) << std::endl;
+    } catch (const parsing_error& error) {
+        std::cerr << error.errorLog() << std::endl;
+        throw error;
+    }
+}