Merge branch 'main' into blocks-metadata

.github/workflows/appimage.yml

@@ -43,7 +43,7 @@ jobs:
           UPDATE_INFO: gh-releases-zsync|MihailRis|VoxelEngine-Cpp|latest|*x86_64.AppImage.zsync
         with:
           recipe: dev/AppImageBuilder.yml
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: AppImage
           path: './*.AppImage*'

.github/workflows/windows.yml

@@ -43,7 +43,7 @@ jobs:
         working-directory: ${{ github.workspace }}
       - name: Run tests
         run: ctest --output-on-failure --test-dir build
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: Windows-Build
           path: 'packaged/Release/*'

src/coders/binary_json.cpp

@@ -14,17 +14,24 @@ static void to_binary(ByteBuilder& builder, const Value& value) {
         case Type::none:
             throw std::runtime_error("none value is not implemented");
         case Type::map: {
-            auto bytes = to_binary(std::get<Map_sptr>(value).get());
+            const auto bytes = to_binary(std::get<Map_sptr>(value).get());
             builder.put(bytes.data(), bytes.size());
             break;
         }
         case Type::list:
             builder.put(BJSON_TYPE_LIST);
-            for (auto& element : std::get<List_sptr>(value)->values) {
+            for (const auto& element : std::get<List_sptr>(value)->values) {
                 to_binary(builder, element);
             }
             builder.put(BJSON_END);
             break;
+        case Type::bytes: {
+            const auto& bytes = std::get<ByteBuffer_sptr>(value).get();
+            builder.put(BJSON_TYPE_BYTES);
+            builder.putInt32(bytes->size());
+            builder.put(bytes->data(), bytes->size());
+            break;
+        }
         case Type::integer: {
             auto val = std::get<integer_t>(value);
             if (val >= 0 && val <= 255) {
@@ -113,11 +120,25 @@ static Value value_from_binary(ByteReader& reader) {
             return (typecode - BJSON_TYPE_FALSE) != 0;
         case BJSON_TYPE_STRING:
             return reader.getString();
-        default:
+        case BJSON_TYPE_NULL:
+            return NONE;
+        case BJSON_TYPE_BYTES: {
+            int32_t size = reader.getInt32();
+            if (size < 0) {
                 throw std::runtime_error(
-                "type " + std::to_string(typecode) + " is not supported"
+                    "invalid byte-buffer size "+std::to_string(size));
-            );
             }
+            if (size > reader.remaining()) {
+                throw std::runtime_error(
+                    "buffer_size > remaining_size "+std::to_string(size));
+            }
+            auto bytes = std::make_shared<ByteBuffer>(reader.pointer(), size);
+            reader.skip(size);
+            return bytes;
+        }
+    }
+    throw std::runtime_error(
+        "type support not implemented for <"+std::to_string(typecode)+">");
 }
 
 static std::unique_ptr<List> array_from_binary(ByteReader& reader) {

src/coders/byte_utils.cpp

@@ -11,7 +11,7 @@ void ByteBuilder::put(ubyte b) {
 }
 
 void ByteBuilder::putCStr(const char* str) {
-    size_t size = strlen(str) + 1;
+    size_t size = std::strlen(str) + 1;
     buffer.reserve(buffer.size() + size);
     for (size_t i = 0; i < size; i++) {
         buffer.push_back(str[i]);
@@ -172,7 +172,7 @@ const char* ByteReader::getCString() {
 }
 
 std::string ByteReader::getString() {
-    uint32_t length = (uint32_t)getInt32();
+    uint32_t length = static_cast<uint32_t>(getInt32());
     if (pos + length > size) {
         throw std::runtime_error("buffer underflow");
     }
@@ -186,6 +186,10 @@ bool ByteReader::hasNext() const {
     return pos < size;
 }
 
+size_t ByteReader::remaining() const {
+    return size - pos;
+}
+
 const ubyte* ByteReader::pointer() const {
     return data + pos;
 }

src/coders/byte_utils.hpp

@@ -72,6 +72,8 @@ public:
     std::string getString();
     /// @return true if there is at least one byte remains
     bool hasNext() const;
+    /// @return Number of remaining bytes in buffer
+    size_t remaining() const;
 
     const ubyte* pointer() const;
     void skip(size_t n);

src/coders/json.cpp

@@ -44,6 +44,14 @@ void stringifyObj(
     bool nice
 );
 
+void stringifyArr(
+    const List* list,
+    std::stringstream& ss,
+    int indent,
+    const std::string& indentstr,
+    bool nice
+);
+
 void stringifyValue(
     const Value& value,
     std::stringstream& ss,
@@ -54,26 +62,11 @@ void stringifyValue(
     if (auto map = std::get_if<Map_sptr>(&value)) {
         stringifyObj(map->get(), ss, indent, indentstr, nice);
     } else if (auto listptr = std::get_if<List_sptr>(&value)) {
-        auto list = *listptr;
+        stringifyArr(listptr->get(), ss, indent, indentstr, nice);
-        if (list->size() == 0) {
+    } else if (auto bytesptr = std::get_if<ByteBuffer_sptr>(&value)) {
-            ss << "[]";
+        auto bytes = bytesptr->get();
-            return;
+        ss << "\"" << util::base64_encode(bytes->data(), bytes->size());
-        }
+        ss << "\"";
-        ss << '[';
-        for (uint i = 0; i < list->size(); i++) {
-            Value& value = list->get(i);
-            if (i > 0 || nice) {
-                newline(ss, nice, indent, indentstr);
-            }
-            stringifyValue(value, ss, indent + 1, indentstr, nice);
-            if (i + 1 < list->size()) {
-                ss << ',';
-            }
-        }
-        if (nice) {
-            newline(ss, true, indent - 1, indentstr);
-        }
-        ss << ']';
     } else if (auto flag = std::get_if<bool>(&value)) {
         ss << (*flag ? "true" : "false");
     } else if (auto num = std::get_if<number_t>(&value)) {
@@ -87,6 +80,38 @@ void stringifyValue(
     }
 }
 
+void stringifyArr(
+    const List* list,
+    std::stringstream& ss,
+    int indent,
+    const std::string& indentstr,
+    bool nice
+) {
+    if (list == nullptr) {
+        ss << "nullptr";
+        return;
+    }
+    if (list->values.empty()) {
+        ss << "[]";
+        return;
+    }
+    ss << "[";
+    for (size_t i = 0; i < list->size(); i++) {
+        if (i > 0 || nice) {
+            newline(ss, nice, indent, indentstr);
+        }
+        const Value& value = list->values[i];
+        stringifyValue(value, ss, indent + 1, indentstr, nice);
+        if (i + 1 < list->size()) {
+            ss << ',';
+        }
+    }
+    if (nice) {
+        newline(ss, true, indent - 1, indentstr);
+    }
+    ss << ']';
+}
+
 void stringifyObj(
     const Map* obj,
     std::stringstream& ss,
@@ -103,7 +128,7 @@ void stringifyObj(
         return;
     }
     ss << "{";
-    uint index = 0;
+    size_t index = 0;
     for (auto& entry : obj->values) {
         const std::string& key = entry.first;
         if (index > 0 || nice) {
@@ -131,6 +156,14 @@ std::string json::stringify(
     return ss.str();
 }
 
+std::string json::stringify(
+    const dynamic::List* arr, bool nice, const std::string& indent
+) {
+    std::stringstream ss;
+    stringifyArr(arr, ss, 1, indent, nice);
+    return ss.str();
+}
+
 std::string json::stringify(
     const dynamic::Value& value, bool nice, const std::string& indent
 ) {

src/coders/json.hpp

@@ -14,6 +14,10 @@ namespace json {
         const dynamic::Map* obj, bool nice, const std::string& indent
     );
 
+    std::string stringify(
+        const dynamic::List* arr, bool nice, const std::string& indent
+    );
+
     std::string stringify(
         const dynamic::Value& value, bool nice, const std::string& indent
     );

src/data/dynamic.cpp

@@ -9,11 +9,22 @@ std::ostream& operator<<(std::ostream& stream, const dynamic::Value& value) {
     return stream;
 }
 
+std::ostream& operator<<(std::ostream& stream, const dynamic::Map& value) {
+    stream << json::stringify(&value, false, " ");
+    return stream;
+}
+
 std::ostream& operator<<(std::ostream& stream, const dynamic::Map_sptr& value) {
     stream << json::stringify(value, false, " ");
     return stream;
 }
 
+std::ostream& operator<<(std::ostream& stream, const dynamic::List& value) {
+    stream << json::stringify(&value, false, " ");
+    return stream;
+}
+
+
 std::ostream& operator<<(
     std::ostream& stream, const dynamic::List_sptr& value
 ) {
@@ -118,6 +129,12 @@ Map& List::putMap() {
     return *map;
 }
 
+ByteBuffer& List::putBytes(size_t size) {
+    auto bytes = create_bytes(size);
+    put(bytes);
+    return *bytes;
+}
+
 void List::remove(size_t index) {
     values.erase(values.begin() + index);
 }
@@ -238,6 +255,12 @@ List_sptr Map::list(const std::string& key) const {
     return nullptr;
 }
 
+ByteBuffer_sptr Map::bytes(const std::string& key) const {
+    auto found = values.find(key);
+    if (found != values.end()) return std::get<ByteBuffer_sptr>(found->second);
+    return nullptr;
+}
+
 void Map::flag(const std::string& key, bool& dst) const {
     dst = get(key, dst);
 }
@@ -263,6 +286,12 @@ Map& Map::putMap(const std::string& key) {
     return *obj;
 }
 
+ByteBuffer& Map::putBytes(const std::string& key, size_t size) {
+    auto bytes = create_bytes(size);
+    put(key, bytes);
+    return *bytes;
+}
+
 bool Map::has(const std::string& key) const {
     return values.find(key) != values.end();
 }
@@ -295,6 +324,10 @@ Map_sptr dynamic::create_map(
     return std::make_shared<Map>(entries);
 }
 
+ByteBuffer_sptr dynamic::create_bytes(size_t size) {
+    return std::make_shared<ByteBuffer>(size);
+}
+
 number_t dynamic::get_number(const Value& value) {
     if (auto num = std::get_if<number_t>(&value)) {
         return *num;

src/data/dynamic.hpp

@@ -11,13 +11,24 @@
 #include "dynamic_fwd.hpp"
 
 namespace dynamic {
-    enum class Type { none = 0, map, list, string, number, boolean, integer };
+    enum class Type {
+        none = 0,
+        map,
+        list,
+        bytes,
+        string,
+        number,
+        boolean,
+        integer
+    };
 
     const std::string& type_name(const Value& value);
     List_sptr create_list(std::initializer_list<Value> values = {});
     Map_sptr create_map(
         std::initializer_list<std::pair<const std::string, Value>> entries = {}
     );
+    ByteBuffer_sptr create_bytes(size_t size);
+
     number_t get_number(const Value& value);
     integer_t get_integer(const Value& value);
 
@@ -59,10 +70,13 @@ namespace dynamic {
         }
 
         List& put(std::unique_ptr<Map> value) {
-            return put(Map_sptr(value.release()));
+            return put(Map_sptr(std::move(value)));
         }
         List& put(std::unique_ptr<List> value) {
-            return put(List_sptr(value.release()));
+            return put(List_sptr(std::move(value)));
+        }
+        List& put(std::unique_ptr<ByteBuffer> value) {
+            return put(ByteBuffer_sptr(std::move(value)));
         }
         List& put(const Value& value);
 
@@ -70,6 +84,7 @@ namespace dynamic {
 
         List& putList();
         Map& putMap();
+        ByteBuffer& putBytes(size_t size);
 
         void remove(size_t index);
     };
@@ -115,6 +130,7 @@ namespace dynamic {
         void num(const std::string& key, double& dst) const;
         Map_sptr map(const std::string& key) const;
         List_sptr list(const std::string& key) const;
+        ByteBuffer_sptr bytes(const std::string& key) const;
         void flag(const std::string& key, bool& dst) const;
 
         Map& put(std::string key, std::unique_ptr<Map> value) {
@@ -144,6 +160,13 @@ namespace dynamic {
         Map& put(std::string key, bool value) {
             return put(key, Value(static_cast<bool>(value)));
         }
+        Map& put(const std::string& key, const ByteBuffer* bytes) {
+            return put(key, std::make_unique<ByteBuffer>(
+                bytes->data(), bytes->size()));
+        }
+        Map& put(std::string key, const ubyte* bytes, size_t size) {
+            return put(key, std::make_unique<ByteBuffer>(bytes, size));
+        }
         Map& put(std::string key, const char* value) {
             return put(key, Value(value));
         }
@@ -153,6 +176,7 @@ namespace dynamic {
 
         List& putList(const std::string& key);
         Map& putMap(const std::string& key);
+        ByteBuffer& putBytes(const std::string& key, size_t size);
 
         bool has(const std::string& key) const;
         size_t size() const;
@@ -160,5 +184,7 @@ namespace dynamic {
 }
 
 std::ostream& operator<<(std::ostream& stream, const dynamic::Value& value);
+std::ostream& operator<<(std::ostream& stream, const dynamic::Map& value);
 std::ostream& operator<<(std::ostream& stream, const dynamic::Map_sptr& value);
+std::ostream& operator<<(std::ostream& stream, const dynamic::List& value);
 std::ostream& operator<<(std::ostream& stream, const dynamic::List_sptr& value);

src/data/dynamic_fwd.hpp

@@ -6,13 +6,16 @@
 #include <variant>
 
 #include "typedefs.hpp"
+#include "util/Buffer.hpp"
 
 namespace dynamic {
     class Map;
     class List;
 
+    using ByteBuffer = util::Buffer<ubyte>;
     using Map_sptr = std::shared_ptr<Map>;
     using List_sptr = std::shared_ptr<List>;
+    using ByteBuffer_sptr = std::shared_ptr<ByteBuffer>;
 
     struct none {};
 
@@ -22,6 +25,7 @@ namespace dynamic {
         none,
         Map_sptr,
         List_sptr,
+        ByteBuffer_sptr,
         std::string,
         number_t,
         bool,

src/graphics/core/Batch3D.cpp

@@ -245,12 +245,18 @@ void Batch3D::blockCube(
     cube((1.0f - size) * -0.5f, size, texfaces, tint, shading);
 }
 
-void Batch3D::point(glm::vec3 coord, glm::vec2 uv, glm::vec4 tint) {
+void Batch3D::vertex(glm::vec3 coord, glm::vec2 uv, glm::vec4 tint) {
+    if (index + B3D_VERTEX_SIZE >= capacity) {
+        flush();
+    }
     vertex(coord, uv, tint.r, tint.g, tint.b, tint.a);
 }
 
 void Batch3D::point(glm::vec3 coord, glm::vec4 tint) {
-    point(coord, glm::vec2(), tint);
+    if (index + B3D_VERTEX_SIZE >= capacity) {
+        flushPoints();
+    }
+    vertex(coord, {}, tint.r, tint.g, tint.b, tint.a);
 }
 
 void Batch3D::flush() {

src/graphics/core/Batch3D.hpp

@@ -52,7 +52,7 @@ public:
     void xSprite(float w, float h, const UVRegion& uv, const glm::vec4 tint, bool shading=true);
     void cube(const glm::vec3 coords, const glm::vec3 size, const UVRegion(&texfaces)[6], const glm::vec4 tint, bool shading=true);
     void blockCube(const glm::vec3 size, const UVRegion(&texfaces)[6], const glm::vec4 tint, bool shading=true);
-    void point(glm::vec3 pos, glm::vec2 uv, glm::vec4 tint);
+    void vertex(glm::vec3 pos, glm::vec2 uv, glm::vec4 tint);
     void point(glm::vec3 pos, glm::vec4 tint);
     void flush() override;
     void flushPoints();

src/graphics/render/BlocksPreview.cpp

@@ -91,12 +91,12 @@ std::unique_ptr<ImageData> BlocksPreview::draw(
                 for (size_t i = 0; i < def.modelExtraPoints.size() / 4; i++) {
                     const UVRegion& reg = def.modelUVs[def.modelBoxes.size() * 6 + i];
                     
-                    batch->point((points[i * 4 + 0] - poff) * pmul, glm::vec2(reg.u1, reg.v1), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 0] - poff) * pmul, glm::vec2(reg.u1, reg.v1), glm::vec4(1.0));
-                    batch->point((points[i * 4 + 1] - poff) * pmul, glm::vec2(reg.u2, reg.v1), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 1] - poff) * pmul, glm::vec2(reg.u2, reg.v1), glm::vec4(1.0));
-                    batch->point((points[i * 4 + 2] - poff) * pmul, glm::vec2(reg.u2, reg.v2), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 2] - poff) * pmul, glm::vec2(reg.u2, reg.v2), glm::vec4(1.0));
-                    batch->point((points[i * 4 + 0] - poff) * pmul, glm::vec2(reg.u1, reg.v1), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 0] - poff) * pmul, glm::vec2(reg.u1, reg.v1), glm::vec4(1.0));
-                    batch->point((points[i * 4 + 2] - poff) * pmul, glm::vec2(reg.u2, reg.v2), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 2] - poff) * pmul, glm::vec2(reg.u2, reg.v2), glm::vec4(1.0));
-                    batch->point((points[i * 4 + 3] - poff) * pmul, glm::vec2(reg.u1, reg.v2), glm::vec4(1.0));
+                    batch->vertex((points[i * 4 + 3] - poff) * pmul, glm::vec2(reg.u1, reg.v2), glm::vec4(1.0));
                 }
                 batch->flush();
             }

src/graphics/ui/elements/Container.cpp

@@ -87,14 +87,15 @@ void Container::draw(const DrawContext* pctx, Assets* assets) {
 
     auto batch = pctx->getBatch2D();
     batch->texture(nullptr);
+    if (!nodes.empty()) {
         batch->flush();
-    {
         DrawContext ctx = pctx->sub();
         ctx.setScissors(glm::vec4(pos.x, pos.y, size.x, size.y));
         for (const auto& node : nodes) {
             if (node->isVisible())
                 node->draw(pctx, assets);
         }
+        batch->flush();
     }
 }
 

src/util/Buffer.hpp

@@ -48,5 +48,9 @@ namespace util {
         Buffer clone() const {
             return Buffer(ptr.get(), length);
         }
+
+        void resizeFast(size_t size) {
+            length = size;
+        }
     };
 }

src/util/stringutil.cpp

@@ -319,8 +319,8 @@ std::string util::mangleid(uint64_t value) {
     return ss.str();
 }
 
-std::vector<ubyte> util::base64_decode(const char* str, size_t size) {
+util::Buffer<ubyte> util::base64_decode(const char* str, size_t size) {
-    std::vector<ubyte> bytes((size / 4) * 3);
+    util::Buffer<ubyte> bytes((size / 4) * 3);
     ubyte* dst = bytes.data();
     for (size_t i = 0; i < size;) {
         ubyte a = base64_decode_char(ubyte(str[i++]));
@@ -335,12 +335,12 @@ std::vector<ubyte> util::base64_decode(const char* str, size_t size) {
         size_t outsize = bytes.size();
         if (str[size - 1] == '=') outsize--;
         if (str[size - 2] == '=') outsize--;
-        bytes.resize(outsize);
+        bytes.resizeFast(outsize);
     }
     return bytes;
 }
 
-std::vector<ubyte> util::base64_decode(const std::string& str) {
+util::Buffer<ubyte> util::base64_decode(const std::string& str) {
     return base64_decode(str.c_str(), str.size());
 }
 

src/util/stringutil.hpp

@@ -4,6 +4,7 @@
 #include <vector>
 
 #include "typedefs.hpp"
+#include "util/Buffer.hpp"
 
 namespace util {
     /// @brief Function used for string serialization in text formats
@@ -56,8 +57,8 @@ namespace util {
     std::wstring to_wstring(double x, int precision);
 
     std::string base64_encode(const ubyte* data, size_t size);
-    std::vector<ubyte> base64_decode(const char* str, size_t size);
+    util::Buffer<ubyte> base64_decode(const char* str, size_t size);
-    std::vector<ubyte> base64_decode(const std::string& str);
+    util::Buffer<ubyte> base64_decode(const std::string& str);
 
     std::string mangleid(uint64_t value);
 

test/coders/binary_json.cpp

@@ -0,0 +1,38 @@
+#include <gtest/gtest.h>
+
+#include "data/dynamic.hpp"
+#include "coders/binary_json.hpp"
+
+TEST(BJSON, EncodeDecode) {
+    const std::string name = "JSON-encoder";
+    const int bytesSize = 5000;
+    const int year = 2019;
+    const float score = 3.141592;
+    dynamic::ByteBuffer srcBytes(bytesSize);
+    for (int i = 0; i < bytesSize; i ++) {
+        srcBytes[i] = rand();
+    }
+
+    std::vector<ubyte> bjsonBytes;
+    {
+        dynamic::Map map;
+        map.put("name", name);
+        map.put("year", year);
+        map.put("score", score);
+        map.put("data", &srcBytes);
+
+        bjsonBytes = json::to_binary(&map, false);
+    }
+    {
+        auto map = json::from_binary(bjsonBytes.data(), bjsonBytes.size());
+        EXPECT_EQ(map->get<std::string>("name"), name);
+        EXPECT_EQ(map->get<integer_t>("year"), year);
+        EXPECT_FLOAT_EQ(map->get<number_t>("score"), score);
+        auto bytesptr = map->bytes("data");
+        const auto& bytes = *bytesptr;
+        EXPECT_EQ(bytes.size(), bytesSize);
+        for (int i = 0; i < bytesSize; i++) {
+            EXPECT_EQ(bytes[i], srcBytes[i]);
+        }
+    }
+}

test/coders/json.cpp

@@ -0,0 +1,39 @@
+#include <gtest/gtest.h>
+
+#include "coders/json.hpp"
+#include "util/stringutil.hpp"
+
+TEST(JSON, EncodeDecode) {
+    const std::string name = "JSON-encoder";
+    const int bytesSize = 20;
+    const int year = 2019;
+    const float score = 3.141592;
+    dynamic::ByteBuffer srcBytes(bytesSize);
+    for (int i = 0; i < bytesSize; i ++) {
+        srcBytes[i] = rand();
+    }
+
+    std::string text;
+    {
+        dynamic::Map map;
+        map.put("name", name);
+        map.put("year", year);
+        map.put("score", score);
+        map.put("data", &srcBytes);
+
+        text = json::stringify(&map, false, "");
+    }
+    {
+        auto map = json::parse(text);
+        EXPECT_EQ(map->get<std::string>("name"), name);
+        EXPECT_EQ(map->get<integer_t>("year"), year);
+        EXPECT_FLOAT_EQ(map->get<number_t>("score"), score);
+        auto b64string = map->get<std::string>("data");
+
+        auto bytes = util::base64_decode(b64string);
+        EXPECT_EQ(bytes.size(), bytesSize);
+        for (int i = 0; i < bytesSize; i++) {
+            EXPECT_EQ(bytes[i], srcBytes[i]);
+        }
+    }
+}