Merge branch 'main' into heightmaps

src/coders/binary_json.cpp

@@ -14,17 +14,24 @@ static void to_binary(ByteBuilder& builder, const Value& value) {
         case Type::none:
             throw std::runtime_error("none value is not implemented");
         case Type::map: {
-            auto bytes = to_binary(std::get<Map_sptr>(value).get());
+            const auto bytes = to_binary(std::get<Map_sptr>(value).get());
             builder.put(bytes.data(), bytes.size());
             break;
         }
         case Type::list:
             builder.put(BJSON_TYPE_LIST);
-            for (auto& element : std::get<List_sptr>(value)->values) {
+            for (const auto& element : std::get<List_sptr>(value)->values) {
                 to_binary(builder, element);
             }
             builder.put(BJSON_END);
             break;
+        case Type::bytes: {
+            const auto& bytes = std::get<ByteBuffer_sptr>(value).get();
+            builder.put(BJSON_TYPE_BYTES);
+            builder.putInt32(bytes->size());
+            builder.put(bytes->data(), bytes->size());
+            break;
+        }
         case Type::integer: {
             auto val = std::get<integer_t>(value);
             if (val >= 0 && val <= 255) {
@@ -113,11 +120,25 @@ static Value value_from_binary(ByteReader& reader) {
             return (typecode - BJSON_TYPE_FALSE) != 0;
         case BJSON_TYPE_STRING:
             return reader.getString();
-        default:
+        case BJSON_TYPE_NULL:
-            throw std::runtime_error(
+            return NONE;
-                "type " + std::to_string(typecode) + " is not supported"
+        case BJSON_TYPE_BYTES: {
-            );
+            int32_t size = reader.getInt32();
+            if (size < 0) {
+                throw std::runtime_error(
+                    "invalid byte-buffer size "+std::to_string(size));
+            }
+            if (size > reader.remaining()) {
+                throw std::runtime_error(
+                    "buffer_size > remaining_size "+std::to_string(size));
+            }
+            auto bytes = std::make_shared<ByteBuffer>(reader.pointer(), size);
+            reader.skip(size);
+            return bytes;
+        }
     }
+    throw std::runtime_error(
+        "type support not implemented for <"+std::to_string(typecode)+">");
 }
 
 static std::unique_ptr<List> array_from_binary(ByteReader& reader) {

src/coders/byte_utils.cpp

@@ -9,7 +9,7 @@ void ByteBuilder::put(ubyte b) {
 }
 
 void ByteBuilder::putCStr(const char* str) {
-    size_t size = strlen(str) + 1;
+    size_t size = std::strlen(str) + 1;
     buffer.reserve(buffer.size() + size);
     for (size_t i = 0; i < size; i++) {
         buffer.push_back(str[i]);
@@ -188,7 +188,7 @@ const char* ByteReader::getCString() {
 }
 
 std::string ByteReader::getString() {
-    uint32_t length = (uint32_t)getInt32();
+    uint32_t length = static_cast<uint32_t>(getInt32());
     if (pos + length > size) {
         throw std::runtime_error("buffer underflow");
     }
@@ -202,6 +202,10 @@ bool ByteReader::hasNext() const {
     return pos < size;
 }
 
+size_t ByteReader::remaining() const {
+    return size - pos;
+}
+
 const ubyte* ByteReader::pointer() const {
     return data + pos;
 }

src/coders/byte_utils.hpp

@@ -74,6 +74,8 @@ public:
     std::string getString();
     /// @return true if there is at least one byte remains
     bool hasNext() const;
+    /// @return Number of remaining bytes in buffer
+    size_t remaining() const;
 
     const ubyte* pointer() const;
     void skip(size_t n);

src/coders/json.cpp

@@ -63,6 +63,10 @@ void stringifyValue(
         stringifyObj(map->get(), ss, indent, indentstr, nice);
     } else if (auto listptr = std::get_if<List_sptr>(&value)) {
         stringifyArr(listptr->get(), ss, indent, indentstr, nice);
+    } else if (auto bytesptr = std::get_if<ByteBuffer_sptr>(&value)) {
+        auto bytes = bytesptr->get();
+        ss << "\"" << util::base64_encode(bytes->data(), bytes->size());
+        ss << "\"";
     } else if (auto flag = std::get_if<bool>(&value)) {
         ss << (*flag ? "true" : "false");
     } else if (auto num = std::get_if<number_t>(&value)) {

src/data/dynamic.cpp

@@ -249,6 +249,12 @@ List_sptr Map::list(const std::string& key) const {
     return nullptr;
 }
 
+ByteBuffer_sptr Map::bytes(const std::string& key) const {
+    auto found = values.find(key);
+    if (found != values.end()) return std::get<ByteBuffer_sptr>(found->second);
+    return nullptr;
+}
+
 void Map::flag(const std::string& key, bool& dst) const {
     dst = get(key, dst);
 }

src/data/dynamic.hpp

@@ -11,7 +11,16 @@
 #include "dynamic_fwd.hpp"
 
 namespace dynamic {
-    enum class Type { none = 0, map, list, string, number, boolean, integer };
+    enum class Type {
+        none = 0,
+        map,
+        list,
+        bytes,
+        string,
+        number,
+        boolean,
+        integer
+    };
 
     const std::string& type_name(const Value& value);
     List_sptr create_list(std::initializer_list<Value> values = {});
@@ -59,10 +68,13 @@ namespace dynamic {
         }
 
         List& put(std::unique_ptr<Map> value) {
-            return put(Map_sptr(value.release()));
+            return put(Map_sptr(std::move(value)));
         }
         List& put(std::unique_ptr<List> value) {
-            return put(List_sptr(value.release()));
+            return put(List_sptr(std::move(value)));
+        }
+        List& put(std::unique_ptr<ByteBuffer> value) {
+            return put(ByteBuffer_sptr(std::move(value)));
         }
         List& put(const Value& value);
 
@@ -115,6 +127,7 @@ namespace dynamic {
         void num(const std::string& key, double& dst) const;
         Map_sptr map(const std::string& key) const;
         List_sptr list(const std::string& key) const;
+        ByteBuffer_sptr bytes(const std::string& key) const;
         void flag(const std::string& key, bool& dst) const;
 
         Map& put(std::string key, std::unique_ptr<Map> value) {
@@ -144,6 +157,13 @@ namespace dynamic {
         Map& put(std::string key, bool value) {
             return put(key, Value(static_cast<bool>(value)));
         }
+        Map& put(const std::string& key, const ByteBuffer* bytes) {
+            return put(key, std::make_unique<ByteBuffer>(
+                bytes->data(), bytes->size()));
+        }
+        Map& put(std::string key, const ubyte* bytes, size_t size) {
+            return put(key, std::make_unique<ByteBuffer>(bytes, size));
+        }
         Map& put(std::string key, const char* value) {
             return put(key, Value(value));
         }

src/data/dynamic_fwd.hpp

@@ -6,13 +6,16 @@
 #include <variant>
 
 #include "typedefs.hpp"
+#include "util/Buffer.hpp"
 
 namespace dynamic {
     class Map;
     class List;
 
+    using ByteBuffer = util::Buffer<ubyte>;
     using Map_sptr = std::shared_ptr<Map>;
     using List_sptr = std::shared_ptr<List>;
+    using ByteBuffer_sptr = std::shared_ptr<ByteBuffer>;
 
     struct none {};
 
@@ -22,6 +25,7 @@ namespace dynamic {
         none,
         Map_sptr,
         List_sptr,
+        ByteBuffer_sptr,
         std::string,
         number_t,
         bool,

src/util/Buffer.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <memory>
+#include <cstring>
+
+namespace util {
+    template<typename T>
+    class Buffer {
+        std::unique_ptr<T[]> ptr;
+        size_t length;
+    public:
+        Buffer(size_t length)
+         : ptr(std::make_unique<T[]>(length)), length(length) {
+        }
+
+        Buffer(std::unique_ptr<T[]> ptr, size_t length)
+         : ptr(std::move(ptr)), length(length) {}
+
+        Buffer(const T* src, size_t length)
+         : ptr(std::make_unique<T[]>(length)), length(length) {
+            std::memcpy(ptr.get(), src, length);
+        }
+
+        T& operator[](long long index) {
+            return ptr[index];
+        }
+
+        const T& operator[](long long index) const {
+            return ptr[index];
+        }
+
+        T* data() {
+            return ptr.get();
+        }
+
+        const T* data() const {
+            return ptr.get();
+        }
+
+        size_t size() const {
+            return length;
+        }
+
+        std::unique_ptr<T[]> release() {
+            return std::move(ptr);
+        }
+
+        Buffer clone() const {
+            return Buffer(ptr.get(), length);
+        }
+
+        void resizeFast(size_t size) {
+            length = size;
+        }
+    };
+}

src/util/stringutil.cpp

@@ -319,8 +319,8 @@ std::string util::mangleid(uint64_t value) {
     return ss.str();
 }
 
-std::vector<ubyte> util::base64_decode(const char* str, size_t size) {
+util::Buffer<ubyte> util::base64_decode(const char* str, size_t size) {
-    std::vector<ubyte> bytes((size / 4) * 3);
+    util::Buffer<ubyte> bytes((size / 4) * 3);
     ubyte* dst = bytes.data();
     for (size_t i = 0; i < size;) {
         ubyte a = base64_decode_char(ubyte(str[i++]));
@@ -335,12 +335,12 @@ std::vector<ubyte> util::base64_decode(const char* str, size_t size) {
         size_t outsize = bytes.size();
         if (str[size - 1] == '=') outsize--;
         if (str[size - 2] == '=') outsize--;
-        bytes.resize(outsize);
+        bytes.resizeFast(outsize);
     }
     return bytes;
 }
 
-std::vector<ubyte> util::base64_decode(const std::string& str) {
+util::Buffer<ubyte> util::base64_decode(const std::string& str) {
     return base64_decode(str.c_str(), str.size());
 }
 

src/util/stringutil.hpp

@@ -4,6 +4,7 @@
 #include <vector>
 
 #include "typedefs.hpp"
+#include "util/Buffer.hpp"
 
 namespace util {
     /// @brief Function used for string serialization in text formats
@@ -56,8 +57,8 @@ namespace util {
     std::wstring to_wstring(double x, int precision);
 
     std::string base64_encode(const ubyte* data, size_t size);
-    std::vector<ubyte> base64_decode(const char* str, size_t size);
+    util::Buffer<ubyte> base64_decode(const char* str, size_t size);
-    std::vector<ubyte> base64_decode(const std::string& str);
+    util::Buffer<ubyte> base64_decode(const std::string& str);
 
     std::string mangleid(uint64_t value);
 

test/coders/binary_json.cpp

@@ -0,0 +1,38 @@
+#include <gtest/gtest.h>
+
+#include "data/dynamic.hpp"
+#include "coders/binary_json.hpp"
+
+TEST(BJSON, EncodeDecode) {
+    const std::string name = "JSON-encoder";
+    const int bytesSize = 5000;
+    const int year = 2019;
+    const float score = 3.141592;
+    dynamic::ByteBuffer srcBytes(bytesSize);
+    for (int i = 0; i < bytesSize; i ++) {
+        srcBytes[i] = rand();
+    }
+
+    std::vector<ubyte> bjsonBytes;
+    {
+        dynamic::Map map;
+        map.put("name", name);
+        map.put("year", year);
+        map.put("score", score);
+        map.put("data", &srcBytes);
+
+        bjsonBytes = json::to_binary(&map, false);
+    }
+    {
+        auto map = json::from_binary(bjsonBytes.data(), bjsonBytes.size());
+        EXPECT_EQ(map->get<std::string>("name"), name);
+        EXPECT_EQ(map->get<integer_t>("year"), year);
+        EXPECT_FLOAT_EQ(map->get<number_t>("score"), score);
+        auto bytesptr = map->bytes("data");
+        const auto& bytes = *bytesptr;
+        EXPECT_EQ(bytes.size(), bytesSize);
+        for (int i = 0; i < bytesSize; i++) {
+            EXPECT_EQ(bytes[i], srcBytes[i]);
+        }
+    }
+}

test/coders/json.cpp

@@ -0,0 +1,39 @@
+#include <gtest/gtest.h>
+
+#include "coders/json.hpp"
+#include "util/stringutil.hpp"
+
+TEST(JSON, EncodeDecode) {
+    const std::string name = "JSON-encoder";
+    const int bytesSize = 20;
+    const int year = 2019;
+    const float score = 3.141592;
+    dynamic::ByteBuffer srcBytes(bytesSize);
+    for (int i = 0; i < bytesSize; i ++) {
+        srcBytes[i] = rand();
+    }
+
+    std::string text;
+    {
+        dynamic::Map map;
+        map.put("name", name);
+        map.put("year", year);
+        map.put("score", score);
+        map.put("data", &srcBytes);
+
+        text = json::stringify(&map, false, "");
+    }
+    {
+        auto map = json::parse(text);
+        EXPECT_EQ(map->get<std::string>("name"), name);
+        EXPECT_EQ(map->get<integer_t>("year"), year);
+        EXPECT_FLOAT_EQ(map->get<number_t>("score"), score);
+        auto b64string = map->get<std::string>("data");
+
+        auto bytes = util::base64_decode(b64string);
+        EXPECT_EQ(bytes.size(), bytesSize);
+        for (int i = 0; i < bytesSize; i++) {
+            EXPECT_EQ(bytes[i], srcBytes[i]);
+        }
+    }
+}