From 8b947fe78e785eaea1e032c445dafb02e3cd84ea Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 12 Sep 2024 13:43:08 +0300 Subject: [PATCH 1/4] add util::Buffer template --- src/util/Buffer.hpp | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/util/Buffer.hpp diff --git a/src/util/Buffer.hpp b/src/util/Buffer.hpp new file mode 100644 index 00000000..4787d2ed --- /dev/null +++ b/src/util/Buffer.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +namespace util { + template + class Buffer { + std::unique_ptr ptr; + size_t length; + public: + Buffer(size_t length) + : ptr(std::make_unique(length)), length(length) { + } + + Buffer(std::unique_ptr ptr, size_t length) + : ptr(std::move(ptr)), length(length) {} + + Buffer(const T* src, size_t length) + : ptr(std::make_unique(length)), length(length) { + std::memcpy(ptr.get(), src, length); + } + + T& operator[](long long index) { + return ptr[index]; + } + + const T& operator[](long long index) const { + return ptr[index]; + } + + T* data() { + return ptr.get(); + } + + const T* data() const { + return ptr.get(); + } + + size_t size() const { + return length; + } + + std::unique_ptr release() { + return std::move(ptr); + } + + Buffer clone() const { + return Buffer(ptr.get(), length); + } + }; +} From 6f4a7db910c24188378fc7dc952bb6d72676b4ff Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 12 Sep 2024 16:24:51 +0300 Subject: [PATCH 2/4] add binary_json BJSON_TYPE_BYTES support --- src/coders/binary_json.cpp | 31 +++++++++++++++++++++++++------ src/data/dynamic.hpp | 18 +++++++++++++++--- src/data/dynamic_fwd.hpp | 4 ++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/coders/binary_json.cpp b/src/coders/binary_json.cpp index 9b8bac22..026593b7 100644 --- a/src/coders/binary_json.cpp +++ b/src/coders/binary_json.cpp @@ -14,17 +14,24 @@ static void to_binary(ByteBuilder& builder, const Value& value) { case Type::none: throw std::runtime_error("none value is not implemented"); case Type::map: { - auto bytes = to_binary(std::get(value).get()); + const auto bytes = to_binary(std::get(value).get()); builder.put(bytes.data(), bytes.size()); break; } case Type::list: builder.put(BJSON_TYPE_LIST); - for (auto& element : std::get(value)->values) { + for (const auto& element : std::get(value)->values) { to_binary(builder, element); } builder.put(BJSON_END); break; + case Type::bytes: { + const auto bytes = std::get(value).get(); + builder.put(BJSON_TYPE_BYTES); + builder.putInt32(bytes->size()); + builder.put(bytes->data(), bytes->size()); + break; + } case Type::integer: { auto val = std::get(value); if (val >= 0 && val <= 255) { @@ -113,11 +120,23 @@ static Value value_from_binary(ByteReader& reader) { return (typecode - BJSON_TYPE_FALSE) != 0; case BJSON_TYPE_STRING: return reader.getString(); - default: - throw std::runtime_error( - "type " + std::to_string(typecode) + " is not supported" - ); + case BJSON_TYPE_NULL: + return NONE; + case BJSON_TYPE_BYTES: { + int32_t size = reader.getInt32(); + if (size < 0) { + throw std::runtime_error( + "invalid byte-buffer size "+std::to_string(size)); + } + if (size > reader.remaining()) { + throw std::runtime_error( + "buffer_size > remaining_size "+std::to_string(size)); + } + return std::make_shared(reader.pointer(), size); + } } + throw std::runtime_error( + "type support not implemented for <"+std::to_string(typecode)+">"); } static std::unique_ptr array_from_binary(ByteReader& reader) { diff --git a/src/data/dynamic.hpp b/src/data/dynamic.hpp index 9525a898..838e5fa8 100644 --- a/src/data/dynamic.hpp +++ b/src/data/dynamic.hpp @@ -11,7 +11,16 @@ #include "dynamic_fwd.hpp" namespace dynamic { - enum class Type { none = 0, map, list, string, number, boolean, integer }; + enum class Type { + none = 0, + map, + list, + string, + number, + boolean, + integer, + bytes + }; const std::string& type_name(const Value& value); List_sptr create_list(std::initializer_list values = {}); @@ -59,10 +68,13 @@ namespace dynamic { } List& put(std::unique_ptr value) { - return put(Map_sptr(value.release())); + return put(Map_sptr(std::move(value))); } List& put(std::unique_ptr value) { - return put(List_sptr(value.release())); + return put(List_sptr(std::move(value))); + } + List& put(std::unique_ptr value) { + return put(ByteBuffer_sptr(std::move(value))); } List& put(const Value& value); diff --git a/src/data/dynamic_fwd.hpp b/src/data/dynamic_fwd.hpp index 148e244b..c4f8eb87 100644 --- a/src/data/dynamic_fwd.hpp +++ b/src/data/dynamic_fwd.hpp @@ -6,13 +6,16 @@ #include #include "typedefs.hpp" +#include "util/Buffer.hpp" namespace dynamic { class Map; class List; + using ByteBuffer = util::Buffer; using Map_sptr = std::shared_ptr; using List_sptr = std::shared_ptr; + using ByteBuffer_sptr = std::shared_ptr; struct none {}; @@ -22,6 +25,7 @@ namespace dynamic { none, Map_sptr, List_sptr, + ByteBuffer_sptr, std::string, number_t, bool, From f8fadf8b74524dd803da5b9f3bceed3d8b65c8af Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 12 Sep 2024 17:22:00 +0300 Subject: [PATCH 3/4] add json BJSON_TYPE_BYTES support --- src/coders/json.cpp | 4 ++++ src/data/dynamic.cpp | 6 ++++++ src/data/dynamic.hpp | 12 ++++++++++-- src/util/Buffer.hpp | 4 ++++ src/util/stringutil.cpp | 8 ++++---- src/util/stringutil.hpp | 5 +++-- test/coders/json.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 test/coders/json.cpp diff --git a/src/coders/json.cpp b/src/coders/json.cpp index ea2d33c9..a15a40a2 100644 --- a/src/coders/json.cpp +++ b/src/coders/json.cpp @@ -63,6 +63,10 @@ void stringifyValue( stringifyObj(map->get(), ss, indent, indentstr, nice); } else if (auto listptr = std::get_if(&value)) { stringifyArr(listptr->get(), ss, indent, indentstr, nice); + } else if (auto bytesptr = std::get_if(&value)) { + auto bytes = bytesptr->get(); + ss << "\"" << util::base64_encode(bytes->data(), bytes->size()); + ss << "\""; } else if (auto flag = std::get_if(&value)) { ss << (*flag ? "true" : "false"); } else if (auto num = std::get_if(&value)) { diff --git a/src/data/dynamic.cpp b/src/data/dynamic.cpp index f49a0aee..42100735 100644 --- a/src/data/dynamic.cpp +++ b/src/data/dynamic.cpp @@ -249,6 +249,12 @@ List_sptr Map::list(const std::string& key) const { return nullptr; } +ByteBuffer_sptr Map::bytes(const std::string& key) const { + auto found = values.find(key); + if (found != values.end()) return std::get(found->second); + return nullptr; +} + void Map::flag(const std::string& key, bool& dst) const { dst = get(key, dst); } diff --git a/src/data/dynamic.hpp b/src/data/dynamic.hpp index 838e5fa8..ed7a0786 100644 --- a/src/data/dynamic.hpp +++ b/src/data/dynamic.hpp @@ -15,11 +15,11 @@ namespace dynamic { none = 0, map, list, + bytes, string, number, boolean, - integer, - bytes + integer }; const std::string& type_name(const Value& value); @@ -127,6 +127,7 @@ namespace dynamic { void num(const std::string& key, double& dst) const; Map_sptr map(const std::string& key) const; List_sptr list(const std::string& key) const; + ByteBuffer_sptr bytes(const std::string& key) const; void flag(const std::string& key, bool& dst) const; Map& put(std::string key, std::unique_ptr value) { @@ -156,6 +157,13 @@ namespace dynamic { Map& put(std::string key, bool value) { return put(key, Value(static_cast(value))); } + Map& put(const std::string& key, const ByteBuffer* bytes) { + return put(key, std::make_unique( + bytes->data(), bytes->size())); + } + Map& put(std::string key, const ubyte* bytes, size_t size) { + return put(key, std::make_unique(bytes, size)); + } Map& put(std::string key, const char* value) { return put(key, Value(value)); } diff --git a/src/util/Buffer.hpp b/src/util/Buffer.hpp index 4787d2ed..68e185c6 100644 --- a/src/util/Buffer.hpp +++ b/src/util/Buffer.hpp @@ -48,5 +48,9 @@ namespace util { Buffer clone() const { return Buffer(ptr.get(), length); } + + void resizeFast(size_t size) { + length = size; + } }; } diff --git a/src/util/stringutil.cpp b/src/util/stringutil.cpp index 97ca4786..4452301b 100644 --- a/src/util/stringutil.cpp +++ b/src/util/stringutil.cpp @@ -319,8 +319,8 @@ std::string util::mangleid(uint64_t value) { return ss.str(); } -std::vector util::base64_decode(const char* str, size_t size) { - std::vector bytes((size / 4) * 3); +util::Buffer util::base64_decode(const char* str, size_t size) { + util::Buffer bytes((size / 4) * 3); ubyte* dst = bytes.data(); for (size_t i = 0; i < size;) { ubyte a = base64_decode_char(ubyte(str[i++])); @@ -335,12 +335,12 @@ std::vector util::base64_decode(const char* str, size_t size) { size_t outsize = bytes.size(); if (str[size - 1] == '=') outsize--; if (str[size - 2] == '=') outsize--; - bytes.resize(outsize); + bytes.resizeFast(outsize); } return bytes; } -std::vector util::base64_decode(const std::string& str) { +util::Buffer util::base64_decode(const std::string& str) { return base64_decode(str.c_str(), str.size()); } diff --git a/src/util/stringutil.hpp b/src/util/stringutil.hpp index 38e7cf89..21d58e5e 100644 --- a/src/util/stringutil.hpp +++ b/src/util/stringutil.hpp @@ -4,6 +4,7 @@ #include #include "typedefs.hpp" +#include "util/Buffer.hpp" namespace util { /// @brief Function used for string serialization in text formats @@ -56,8 +57,8 @@ namespace util { std::wstring to_wstring(double x, int precision); std::string base64_encode(const ubyte* data, size_t size); - std::vector base64_decode(const char* str, size_t size); - std::vector base64_decode(const std::string& str); + util::Buffer base64_decode(const char* str, size_t size); + util::Buffer base64_decode(const std::string& str); std::string mangleid(uint64_t value); diff --git a/test/coders/json.cpp b/test/coders/json.cpp new file mode 100644 index 00000000..7212a087 --- /dev/null +++ b/test/coders/json.cpp @@ -0,0 +1,41 @@ +#include + +#include "coders/json.hpp" +#include "util/stringutil.hpp" + +#include + +TEST(JSON, EncodeDecode) { + const std::string name = "JSON-encoder"; + const int bytesSize = 20; + const int year = 2019; + const float score = 3.141592; + dynamic::ByteBuffer srcBytes(bytesSize); + for (int i = 0; i < bytesSize; i ++) { + srcBytes[i] = rand(); + } + + std::string text; + { + dynamic::Map map; + map.put("name", name); + map.put("year", year); + map.put("score", score); + map.put("data", &srcBytes); + + text = json::stringify(&map, false, ""); + } + { + auto map = json::parse(text); + EXPECT_EQ(map->get("name"), name); + EXPECT_EQ(map->get("year"), year); + EXPECT_FLOAT_EQ(map->get("score"), score); + auto b64string = map->get("data"); + + auto bytes = util::base64_decode(b64string); + EXPECT_EQ(bytes.size(), bytesSize); + for (int i = 0; i < bytesSize; i++) { + EXPECT_EQ(bytes[i], srcBytes[i]); + } + } +} From c3fb30b20e293ea3363dba49b7dffe99c60e0027 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 12 Sep 2024 17:39:05 +0300 Subject: [PATCH 4/4] fix bjson decoder --- src/coders/binary_json.cpp | 6 ++++-- test/coders/binary_json.cpp | 38 +++++++++++++++++++++++++++++++++++++ test/coders/json.cpp | 2 -- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 test/coders/binary_json.cpp diff --git a/src/coders/binary_json.cpp b/src/coders/binary_json.cpp index 026593b7..fd5d2c23 100644 --- a/src/coders/binary_json.cpp +++ b/src/coders/binary_json.cpp @@ -26,7 +26,7 @@ static void to_binary(ByteBuilder& builder, const Value& value) { builder.put(BJSON_END); break; case Type::bytes: { - const auto bytes = std::get(value).get(); + const auto& bytes = std::get(value).get(); builder.put(BJSON_TYPE_BYTES); builder.putInt32(bytes->size()); builder.put(bytes->data(), bytes->size()); @@ -132,7 +132,9 @@ static Value value_from_binary(ByteReader& reader) { throw std::runtime_error( "buffer_size > remaining_size "+std::to_string(size)); } - return std::make_shared(reader.pointer(), size); + auto bytes = std::make_shared(reader.pointer(), size); + reader.skip(size); + return bytes; } } throw std::runtime_error( diff --git a/test/coders/binary_json.cpp b/test/coders/binary_json.cpp new file mode 100644 index 00000000..2a3ecea1 --- /dev/null +++ b/test/coders/binary_json.cpp @@ -0,0 +1,38 @@ +#include + +#include "data/dynamic.hpp" +#include "coders/binary_json.hpp" + +TEST(BJSON, EncodeDecode) { + const std::string name = "JSON-encoder"; + const int bytesSize = 5000; + const int year = 2019; + const float score = 3.141592; + dynamic::ByteBuffer srcBytes(bytesSize); + for (int i = 0; i < bytesSize; i ++) { + srcBytes[i] = rand(); + } + + std::vector bjsonBytes; + { + dynamic::Map map; + map.put("name", name); + map.put("year", year); + map.put("score", score); + map.put("data", &srcBytes); + + bjsonBytes = json::to_binary(&map, false); + } + { + auto map = json::from_binary(bjsonBytes.data(), bjsonBytes.size()); + EXPECT_EQ(map->get("name"), name); + EXPECT_EQ(map->get("year"), year); + EXPECT_FLOAT_EQ(map->get("score"), score); + auto bytesptr = map->bytes("data"); + const auto& bytes = *bytesptr; + EXPECT_EQ(bytes.size(), bytesSize); + for (int i = 0; i < bytesSize; i++) { + EXPECT_EQ(bytes[i], srcBytes[i]); + } + } +} diff --git a/test/coders/json.cpp b/test/coders/json.cpp index 7212a087..5738559b 100644 --- a/test/coders/json.cpp +++ b/test/coders/json.cpp @@ -3,8 +3,6 @@ #include "coders/json.hpp" #include "util/stringutil.hpp" -#include - TEST(JSON, EncodeDecode) { const std::string name = "JSON-encoder"; const int bytesSize = 20;