From cf99996b24e868af4eaaf4551945f4b3387e3f98 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 12 Aug 2024 22:07:25 +0300 Subject: [PATCH 01/61] add util::SmallHeap --- src/util/SmallHeap.hpp | 102 ++++++++++++++++++++++++++++++++++++++++ test/util/SmallHeap.cpp | 55 ++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 src/util/SmallHeap.hpp create mode 100644 test/util/SmallHeap.cpp diff --git a/src/util/SmallHeap.hpp b/src/util/SmallHeap.hpp new file mode 100644 index 00000000..ef590ff0 --- /dev/null +++ b/src/util/SmallHeap.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include + +namespace util { + template + class SmallHeap { + std::vector buffer; + Tindex entriesCount; + public: + SmallHeap() : entriesCount(0) {} + + uint8_t* find(Tindex index) { + auto data = buffer.data(); + for (size_t i = 0; i < entriesCount; i++) { + auto nextIndex = *reinterpret_cast(data); + data += sizeof(Tindex); + auto nextSize = *reinterpret_cast(data); + data += sizeof(Tsize); + if (nextIndex == index) { + return data; + } else if (nextIndex > index) { + return nullptr; + } + data += nextSize; + } + return nullptr; + } + + void free(uint8_t* ptr) { + if (ptr == nullptr) { + return; + } + auto entrySize = sizeOf(ptr); + auto begin = + buffer.begin() + + ((ptr - sizeof(Tsize) - sizeof(Tindex)) - buffer.data()); + buffer.erase( + begin, begin + entrySize + sizeof(Tsize) + sizeof(Tindex) + ); + entriesCount--; + } + + uint8_t* allocate(Tindex index, Tsize size) { + if (size == 0) { + throw std::runtime_error("size is 0"); + } + ptrdiff_t offset = 0; + if (auto found = find(index)) { + auto entrySize = sizeOf(found); + if (size == entrySize) { + std::memset(found, 0, entrySize); + return found; + } + this->free(found); + return allocate(index, size); + } + for (size_t i = 0; i < entriesCount; i++) { + auto data = buffer.data() + offset; + auto nextIndex = *reinterpret_cast(data); + data += sizeof(Tindex); + auto nextSize = *reinterpret_cast(data); + data += sizeof(Tsize); + if (nextIndex > index) { + break; + } + data += nextSize; + offset = data - buffer.data(); + } + buffer.insert( + buffer.begin() + offset, + size + sizeof(Tindex) + sizeof(Tsize), + 0 + ); + entriesCount++; + + auto data = buffer.data() + offset; + *reinterpret_cast(data) = index; + data += sizeof(Tindex); + *reinterpret_cast(data) = size; + return data + sizeof(Tsize); + } + + Tsize sizeOf(uint8_t* ptr) const { + if (ptr == nullptr) { + return 0; + } + return *(reinterpret_cast(ptr)-1); + } + + Tindex count() const { + return entriesCount; + } + + size_t size() const { + return buffer.size(); + } + }; +} diff --git a/test/util/SmallHeap.cpp b/test/util/SmallHeap.cpp new file mode 100644 index 00000000..d8b08a4c --- /dev/null +++ b/test/util/SmallHeap.cpp @@ -0,0 +1,55 @@ +#include + +#include "util/SmallHeap.hpp" + +using namespace util; + +TEST(SmallHeap, Allocation) { + auto index = 0; + auto size = 4; + + SmallHeap map; + auto ptr = map.allocate(index, size); + EXPECT_EQ(map.sizeOf(ptr), size); + EXPECT_EQ(ptr, map.find(index)); +} + +TEST(SmallHeap, MultipleAllocation) { + SmallHeap map; + map.allocate(32, 4); + map.allocate(1, 5); + EXPECT_EQ(map.sizeOf(map.find(32)), 4); + EXPECT_EQ(map.sizeOf(map.find(1)), 5); +} + +TEST(SmallHeap, Free) { + SmallHeap map; + map.free(map.allocate(5, 4)); + EXPECT_EQ(map.find(5), nullptr); +} + +TEST(SmallHeap, ReAllocationSame) { + SmallHeap map; + auto ptr1 = map.allocate(1, 2); + auto ptr2 = map.allocate(1, 2); + EXPECT_EQ(ptr1, ptr2); +} + +TEST(SmallHeap, ReAllocationDifferent) { + SmallHeap map; + map.allocate(1, 34); + map.allocate(1, 2); + EXPECT_EQ(map.sizeOf(map.find(1)), 2); +} + +TEST(SmallHeap, RandomFill) { + SmallHeap map; + int n = 3'000; + map.allocate(n, 123); + for (int i = 0; i < n; i++) { + int index = rand() % n; + int size = rand() % 254 + 1; + map.allocate(index, size); + } + EXPECT_EQ(map.sizeOf(map.find(n)), 123); +} From b60796c4b070ff49103be7dbdaa5c8ae4de73bc4 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 13 Aug 2024 01:18:13 +0300 Subject: [PATCH 02/61] add StructMapper draft --- src/data/StructMapper.hpp | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/data/StructMapper.hpp diff --git a/src/data/StructMapper.hpp b/src/data/StructMapper.hpp new file mode 100644 index 00000000..1c92f882 --- /dev/null +++ b/src/data/StructMapper.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +#include "typedefs.hpp" + +namespace data { + enum class FieldType { + I8, I16, I32, I64, F32, F64 + }; + + struct Field { + FieldType type; + std::string name; + int offset; + }; + + class StructMapping { + std::vector fields; + std::unordered_map indices; + public: + const Field* getField(const std::string& name) const { + auto found = indices.find(name); + if (found == indices.end()) { + return nullptr; + } + return &fields.at(found->second); + } + }; + + class StructAccess { + const StructMapping& mapping; + uint8_t* buffer; + public: + StructAccess(const StructMapping& mapping, uint8_t* buffer) + : mapping(mapping), buffer(buffer) { + } + }; +} From 9b87071f2423f4e4e54355fdc0546b96d41d72ab Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 11:37:05 +0300 Subject: [PATCH 03/61] add byteorder-related functions to data_io.hpp --- src/util/data_io.hpp | 63 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/util/data_io.hpp b/src/util/data_io.hpp index c36007b5..02642a54 100644 --- a/src/util/data_io.hpp +++ b/src/util/data_io.hpp @@ -3,16 +3,67 @@ #include "typedefs.hpp" namespace dataio { - /* Read big-endian 16 bit signed integer from bytes */ + /// @brief Swap byte-order for value of type T + /// @tparam T value type + /// @param value source integer + /// @return swapped integer + template + T swap(T value) { + union{ + T value; + ubyte bytes[sizeof(T)]; + } source, dest; + + source.value = value; + + for (size_t i = 0; i < sizeof(T); i++) { + dest.bytes[i] = source.bytes[sizeof(T) - i - 1]; + } + return dest.value; + } + + inline bool is_big_endian() { + uint16_t num = 1; + auto bytes = reinterpret_cast(num); + return bytes[1] == 1; + } + + /// @brief Convert big-endian to hardware byte-order + /// @tparam T value type + /// @param value source integer + /// @return integer with hardware byte-order + template + T be2h(T value){ + if (is_big_endian()) { + return value; + } else { + return swap(value); + } + } + + /// @brief Convert little-endian to hardware byte-order + /// @tparam T value type + /// @param value source integer + /// @return integer with hardware byte-order + template + T le2h(T value){ + if (is_big_endian()) { + return swap(value); + } else { + return value; + } + } + + /// @brief Read big-endian 16 bit signed integer from bytes inline int16_t read_int16_big(const ubyte* src, size_t offset) { return (src[offset] << 8) | (src[offset + 1]); } - /* Read big-endian 32 bit signed integer from bytes */ + /// @brief Read big-endian 32 bit signed integer from bytes inline int32_t read_int32_big(const ubyte* src, size_t offset) { return (src[offset] << 24) | (src[offset + 1] << 16) | (src[offset + 2] << 8) | (src[offset + 3]); } - /* Read big-endian 64 bit signed integer from bytes */ + /// @brief Read big-endian 64 bit signed integer from bytes inline int64_t read_int64_big(const ubyte* src, size_t offset) { return (int64_t(src[offset]) << 56) | (int64_t(src[offset + 1]) << 48) | (int64_t(src[offset + 2]) << 40) | @@ -21,19 +72,19 @@ namespace dataio { (int64_t(src[offset + 5]) << 16) | (int64_t(src[offset + 6]) << 8) | (int64_t(src[offset + 7])); } - /* Write big-endian 16 bit signed integer to bytes */ + /// @brief Write big-endian 16 bit signed integer to bytes inline void write_int16_big(int16_t value, ubyte* dest, size_t offset) { dest[offset] = (char)(value >> 8 & 255); dest[offset + 1] = (char)(value >> 0 & 255); } - /* Write big-endian 32 bit signed integer to bytes */ + /// @brief Write big-endian 32 bit signed integer to bytes inline void write_int32_big(int32_t value, ubyte* dest, size_t offset) { dest[offset] = (char)(value >> 24 & 255); dest[offset + 1] = (char)(value >> 16 & 255); dest[offset + 2] = (char)(value >> 8 & 255); dest[offset + 3] = (char)(value >> 0 & 255); } - /* Write big-endian 64 bit signed integer to bytes */ + /// @brief Write big-endian 64 bit signed integer to bytes inline void write_int64_big(int64_t value, ubyte* dest, size_t offset) { dest[offset] = (char)(value >> 56 & 255); dest[offset + 1] = (char)(value >> 48 & 255); From 2b1db0b075b72ec0d9d4486b15b9a5053c8a1c81 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 11:51:33 +0300 Subject: [PATCH 04/61] fix --- src/util/data_io.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/data_io.hpp b/src/util/data_io.hpp index 02642a54..93022f7d 100644 --- a/src/util/data_io.hpp +++ b/src/util/data_io.hpp @@ -24,7 +24,7 @@ namespace dataio { inline bool is_big_endian() { uint16_t num = 1; - auto bytes = reinterpret_cast(num); + auto bytes = reinterpret_cast(&num); return bytes[1] == 1; } From c67b867a319b78bad233c065585a79040725a4f8 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 11:57:05 +0300 Subject: [PATCH 05/61] add StructMapper read/write methods --- src/data/StructMapper.cpp | 162 +++++++++++++++++++++++++++++++++++++ src/data/StructMapper.hpp | 40 ++++++++- test/data/StructMapper.cpp | 23 ++++++ 3 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 src/data/StructMapper.cpp create mode 100644 test/data/StructMapper.cpp diff --git a/src/data/StructMapper.cpp b/src/data/StructMapper.cpp new file mode 100644 index 00000000..238fd665 --- /dev/null +++ b/src/data/StructMapper.cpp @@ -0,0 +1,162 @@ +#include "StructMapper.hpp" + +#include +#include +#include + +#include "util/data_io.hpp" + +using namespace data; + +static_assert(sizeof(float) == sizeof(int32_t)); +static_assert(sizeof(double) == sizeof(int64_t)); + +StructMapping StructMapping::create(const std::vector& fields) { + std::vector builtFields = fields; + std::unordered_map indices; + + for (Field& field : builtFields) { + field.size = sizeof_type(field.type) * field.elements; + } + std::sort(builtFields.begin(), builtFields.end(), + [](const Field& a, const Field& b) { + return a.size > b.size; + } + ); + int offset = 0; + for (int i = 0; i < builtFields.size(); i++) { + auto& field = builtFields[i]; + field.offset = offset; + indices[field.name] = i; + offset += field.size; + } + return StructMapping( + offset, std::move(builtFields), std::move(indices)); +} + +const Field& StructMapping::requreField(const std::string& name) const { + auto found = indices.find(name); + if (found == indices.end()) { + throw std::runtime_error("field '"+name+"' does not exist"); + } + return *&fields.at(found->second); +} + + +template +static void set_int(ubyte* dst, integer_t value) { + T out_value = static_cast(value); + out_value = dataio::le2h(out_value); + *reinterpret_cast(dst) = out_value; +} + +void StructMapping::setInteger( + ubyte* dst, integer_t value, const std::string& name, int index +) { + const auto& field = requreField(name); + if (index < 0 || index >= field.elements) { + throw std::runtime_error( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = dst + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::I8: set_int(ptr, value); break; + case FieldType::I16: set_int(ptr, value); break; + case FieldType::I32: set_int(ptr, value); break; + case FieldType::I64: set_int(ptr, value); break; + case FieldType::CHAR: set_int(ptr, value); break; + case FieldType::F32: + case FieldType::F64: + setNumber(dst, static_cast(value), name, index); + break; + default: + throw std::runtime_error("type error"); + } +} + +void StructMapping::setNumber( + ubyte* dst, number_t value, const std::string& name, int index +) { + const auto& field = requreField(name); + if (index < 0 || index >= field.elements) { + throw std::runtime_error( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = dst + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::F32: { + float fval = static_cast(value); + int32_t ival; + std::memcpy(&ival, &fval, sizeof(int32_t)); + set_int(ptr, ival); + break; + } + case FieldType::F64: { + double fval = static_cast(value); + int64_t ival; + std::memcpy(&ival, &fval, sizeof(int64_t)); + set_int(ptr, ival); + break; + } + default: + throw std::runtime_error("type error"); + } +} + +template +static T get_int(const ubyte* src) { + return dataio::le2h(*reinterpret_cast(src)); +} + +integer_t StructMapping::getInteger( + const ubyte* src, const std::string& name, int index +) const { + const auto& field = requreField(name); + if (index < 0 || index >= field.elements) { + throw std::runtime_error( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = src + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::I8: return get_int(ptr); + case FieldType::I16: return get_int(ptr); + case FieldType::I32: return get_int(ptr); + case FieldType::I64: return get_int(ptr); + case FieldType::CHAR: return get_int(ptr); + default: + throw std::runtime_error("type error"); + } +} + +number_t StructMapping::getNumber( + const ubyte* src, const std::string& name, int index +) const { + const auto& field = requreField(name); + if (index < 0 || index >= field.elements) { + throw std::runtime_error( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = src + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::F32: { + float fval; + auto ival = get_int(ptr); + std::memcpy(&fval, &ival, sizeof(float)); + return fval; + } + case FieldType::F64: { + double fval; + auto ival = get_int(ptr); + std::memcpy(&fval, &ival, sizeof(double)); + return fval; + } + case FieldType::I8: + case FieldType::I16: + case FieldType::I32: + case FieldType::I64: + case FieldType::CHAR: + return getInteger(src, name, index); + + } + throw std::runtime_error("type error"); +} diff --git a/src/data/StructMapper.hpp b/src/data/StructMapper.hpp index 1c92f882..32c33ed1 100644 --- a/src/data/StructMapper.hpp +++ b/src/data/StructMapper.hpp @@ -8,26 +8,64 @@ namespace data { enum class FieldType { - I8, I16, I32, I64, F32, F64 + I8=0, I16, I32, I64, F32, F64, CHAR, + COUNT }; + inline constexpr int sizeof_type(FieldType type) { + const int sizes[static_cast(FieldType::COUNT)] = { + 1, 2, 4, 8, 4, 8, 1 + }; + return sizes[static_cast(type)]; + } + struct Field { FieldType type; std::string name; + /// @brief Number of field elements (array size) + int elements; + /// @brief Byte offset of the field int offset; + /// @brief Byte size of the field + int size; }; class StructMapping { + int totalSize; std::vector fields; std::unordered_map indices; public: + StructMapping( + int totalSize, + std::vector fields, + std::unordered_map indices + ) : totalSize(totalSize), + fields(std::move(fields)), + indices(std::move(indices)) + {} + const Field* getField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { return nullptr; } + return &fields.at(found->second); } + + const Field& requreField(const std::string& name) const; + + integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const; + number_t getNumber(const ubyte* src, const std::string& name, int index=0) const; + + void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0); + void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0); + + int size() const { + return totalSize; + } + + static StructMapping create(const std::vector& fields); }; class StructAccess { diff --git a/test/data/StructMapper.cpp b/test/data/StructMapper.cpp new file mode 100644 index 00000000..60c874b1 --- /dev/null +++ b/test/data/StructMapper.cpp @@ -0,0 +1,23 @@ +#include "data/StructMapper.hpp" + +#include + +using namespace data; + +TEST(StructMapper, ReadWrite) { + ubyte buffer[16] {}; + + std::vector fields { + Field {FieldType::I8, "a", 1}, + Field {FieldType::I32, "b", 1}, + Field {FieldType::F32, "f", 1}, + }; + auto mapping = StructMapping::create(fields); + EXPECT_EQ(mapping.size(), 9); + + mapping.setInteger(buffer, 42, "a"); + EXPECT_EQ(mapping.getInteger(buffer, "a"), 42); + + mapping.setNumber(buffer, 3.141592f, "f"); + EXPECT_FLOAT_EQ(mapping.getNumber(buffer, "f"), 3.141592f); +} From b34fddbe945339bd41394578c68bfa61e9b5d1b1 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 12:32:41 +0300 Subject: [PATCH 06/61] add StructMapper.setChars, getChars --- src/data/StructMapper.cpp | 23 +++++++++++++++++++++++ src/data/StructMapper.hpp | 2 ++ test/data/StructMapper.cpp | 5 ++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/data/StructMapper.cpp b/src/data/StructMapper.cpp index 238fd665..1e26f816 100644 --- a/src/data/StructMapper.cpp +++ b/src/data/StructMapper.cpp @@ -103,6 +103,18 @@ void StructMapping::setNumber( } } +void StructMapping::setChars( + ubyte* dst, std::string_view value, const std::string& name +) { + const auto& field = requreField(name); + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto ptr = reinterpret_cast(dst + field.offset); + std::memcpy(ptr, value.data(), + std::min(value.size(), static_cast(field.elements))); +} + template static T get_int(const ubyte* src) { return dataio::le2h(*reinterpret_cast(src)); @@ -160,3 +172,14 @@ number_t StructMapping::getNumber( } throw std::runtime_error("type error"); } + +std::string_view StructMapping::getChars( + const ubyte* src, const std::string& name +) const { + const auto& field = requreField(name); + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto ptr = src + field.offset; + return std::string_view(reinterpret_cast(ptr), field.elements); +} diff --git a/src/data/StructMapper.hpp b/src/data/StructMapper.hpp index 32c33ed1..358105f9 100644 --- a/src/data/StructMapper.hpp +++ b/src/data/StructMapper.hpp @@ -57,9 +57,11 @@ namespace data { integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const; number_t getNumber(const ubyte* src, const std::string& name, int index=0) const; + std::string_view getChars(const ubyte* src, const std::string& name) const; void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0); void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0); + void setChars(ubyte* dst, std::string_view value, const std::string& name); int size() const { return totalSize; diff --git a/test/data/StructMapper.cpp b/test/data/StructMapper.cpp index 60c874b1..67aeb97b 100644 --- a/test/data/StructMapper.cpp +++ b/test/data/StructMapper.cpp @@ -9,7 +9,7 @@ TEST(StructMapper, ReadWrite) { std::vector fields { Field {FieldType::I8, "a", 1}, - Field {FieldType::I32, "b", 1}, + Field {FieldType::CHAR, "s", 4}, Field {FieldType::F32, "f", 1}, }; auto mapping = StructMapping::create(fields); @@ -20,4 +20,7 @@ TEST(StructMapper, ReadWrite) { mapping.setNumber(buffer, 3.141592f, "f"); EXPECT_FLOAT_EQ(mapping.getNumber(buffer, "f"), 3.141592f); + + mapping.setChars(buffer, "hello", "s"); + EXPECT_EQ(mapping.getChars(buffer, "s"), "hell"); } From 25bdcc8e1bbb579ba6fb4c2e08443fd56b914d06 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 13:01:32 +0300 Subject: [PATCH 07/61] add util::crop_utf8 (stringutil) --- src/util/stringutil.cpp | 13 +++++++++++++ src/util/stringutil.hpp | 15 +++++++++++++++ test/util/stringutil.cpp | 10 ++++++++++ 3 files changed, 38 insertions(+) create mode 100644 test/util/stringutil.cpp diff --git a/src/util/stringutil.cpp b/src/util/stringutil.cpp index c494a759..8f3bddd8 100644 --- a/src/util/stringutil.cpp +++ b/src/util/stringutil.cpp @@ -141,6 +141,19 @@ extern uint32_t util::decode_utf8(uint& size, const char* chr) { return code; } +size_t util::crop_utf8(std::string_view s, size_t maxSize) { + size_t pos = 0; + uint size = 0; + while (pos < s.length()) { + decode_utf8(size, &s.at(pos)); + if (pos + size > maxSize) { + return pos; + } + pos += size; + } + return pos; +} + std::string util::wstr2str_utf8(const std::wstring& ws) { std::vector chars; char buffer[4]; diff --git a/src/util/stringutil.hpp b/src/util/stringutil.hpp index 54a0a5e2..14ba9432 100644 --- a/src/util/stringutil.hpp +++ b/src/util/stringutil.hpp @@ -17,8 +17,23 @@ namespace util { uint encode_utf8(uint32_t c, ubyte* bytes); uint32_t decode_utf8(uint& size, const char* bytes); + + /// @brief Encode raw wstring to UTF-8 + /// @param ws source raw wstring + /// @return new UTF-8 encoded string std::string wstr2str_utf8(const std::wstring& ws); + + /// @brief Decode UTF + /// @param s source encoded string + /// @return new raw decoded string std::wstring str2wstr_utf8(const std::string& s); + + /// @brief Calculated length of UTF-8 encoded string that fits into maxSize + /// @param s source UTF-8 encoded string view + /// @param maxSize max encoded string length after crop + /// @return cropped string size (less or equal to maxSize) + size_t crop_utf8(std::string_view s, size_t maxSize); + bool is_integer(const std::string& text); bool is_integer(const std::wstring& text); bool is_valid_filename(const std::wstring& name); diff --git a/test/util/stringutil.cpp b/test/util/stringutil.cpp new file mode 100644 index 00000000..d2172380 --- /dev/null +++ b/test/util/stringutil.cpp @@ -0,0 +1,10 @@ +#include "util/stringutil.hpp" + +#include + +TEST(stringutil, crop_utf8) { + // Project source files must be UTF-8 encoded + std::string str = "пример"; + str = str.substr(0, util::crop_utf8(str, 7)); + EXPECT_EQ(str, "при"); +} From d6ff55e4f5508d3584ebbd6b1e38b9e9584fdab2 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 13:32:56 +0300 Subject: [PATCH 08/61] update .github/workflows/windows.yml --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 6af90077..60ad52a6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -42,7 +42,7 @@ jobs: cp -r build/* packaged/ working-directory: ${{ github.workspace }} - name: Run tests - run: ctest --test-dir build + run: ctest --output-on-failure --test-dir build - uses: actions/upload-artifact@v2 with: name: Windows-Build From bd87a586d80c83ead9c068075f08ffb44734e0a6 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 13:47:18 +0300 Subject: [PATCH 09/61] fix stringutil test --- test/util/stringutil.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/util/stringutil.cpp b/test/util/stringutil.cpp index d2172380..ed6e2b00 100644 --- a/test/util/stringutil.cpp +++ b/test/util/stringutil.cpp @@ -4,7 +4,7 @@ TEST(stringutil, crop_utf8) { // Project source files must be UTF-8 encoded - std::string str = "пример"; + std::string str = u8"пример"; str = str.substr(0, util::crop_utf8(str, 7)); - EXPECT_EQ(str, "при"); + EXPECT_EQ(str, u8"при"); } From 4343e81e0069323791deebbcfb692a36bd37e361 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 15:09:27 +0300 Subject: [PATCH 10/61] fix utf-8 decoder & add u32string functions --- src/util/stringutil.cpp | 61 +++++++++++++++++++++++++++------------- src/util/stringutil.hpp | 14 +++++++-- test/util/stringutil.cpp | 7 +++++ 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/src/util/stringutil.cpp b/src/util/stringutil.cpp index 8f3bddd8..97ca4786 100644 --- a/src/util/stringutil.cpp +++ b/src/util/stringutil.cpp @@ -116,20 +116,22 @@ const utf_t utf[] = { }; inline uint utf8_len(ubyte cp) { - uint len = 0; - for (const utf_t* u = utf; u->mask; ++u) { - if ((cp >= u->beg) && (cp <= u->end)) { - break; - } - ++len; + if ((cp & 0x80) == 0) { + return 1; } - if (len > 4) /* Out of bounds */ - throw std::runtime_error("utf-8 decode error"); - - return len; + if ((cp & 0xE0) == 0xC0) { + return 2; + } + if ((cp & 0xF0) == 0xE0) { + return 3; + } + if ((cp & 0xF8) == 0xF0) { + return 4; + } + return 0; } -extern uint32_t util::decode_utf8(uint& size, const char* chr) { +uint32_t util::decode_utf8(uint& size, const char* chr) { size = utf8_len(*chr); int shift = utf[0].bits_stored * (size - 1); uint32_t code = (*chr++ & utf[size].mask) << shift; @@ -145,7 +147,7 @@ size_t util::crop_utf8(std::string_view s, size_t maxSize) { size_t pos = 0; uint size = 0; while (pos < s.length()) { - decode_utf8(size, &s.at(pos)); + decode_utf8(size, s.data() + pos); if (pos + size > maxSize) { return pos; } @@ -154,11 +156,13 @@ size_t util::crop_utf8(std::string_view s, size_t maxSize) { return pos; } -std::string util::wstr2str_utf8(const std::wstring& ws) { +template +std::string xstr2str_utf8(const std::basic_string& xs) { std::vector chars; - char buffer[4]; - for (wchar_t wc : ws) { - uint size = encode_utf8((uint)wc, (ubyte*)buffer); + ubyte buffer[4]; + for (C xc : xs) { + uint size = util::encode_utf8( + static_cast(xc), buffer); for (uint i = 0; i < size; i++) { chars.push_back(buffer[i]); } @@ -166,15 +170,32 @@ std::string util::wstr2str_utf8(const std::wstring& ws) { return std::string(chars.data(), chars.size()); } -std::wstring util::str2wstr_utf8(const std::string& s) { - std::vector chars; +std::string util::wstr2str_utf8(const std::wstring& ws) { + return xstr2str_utf8(ws); +} + +std::string util::u32str2str_utf8(const std::u32string& ws) { + return xstr2str_utf8(ws); +} + +template +std::basic_string str2xstr_utf8(const std::string& s) { + std::vector chars; size_t pos = 0; uint size = 0; while (pos < s.length()) { - chars.push_back(decode_utf8(size, &s.at(pos))); + chars.push_back(util::decode_utf8(size, &s.at(pos))); pos += size; } - return std::wstring(chars.data(), chars.size()); + return std::basic_string(chars.data(), chars.size()); +} + +std::wstring util::str2wstr_utf8(const std::string& s) { + return str2xstr_utf8(s); +} + +std::u32string util::str2u32str_utf8(const std::string& s) { + return str2xstr_utf8(s); } bool util::is_integer(const std::string& text) { diff --git a/src/util/stringutil.hpp b/src/util/stringutil.hpp index 14ba9432..38e7cf89 100644 --- a/src/util/stringutil.hpp +++ b/src/util/stringutil.hpp @@ -23,11 +23,21 @@ namespace util { /// @return new UTF-8 encoded string std::string wstr2str_utf8(const std::wstring& ws); - /// @brief Decode UTF + /// @brief Decode UTF-8 string /// @param s source encoded string - /// @return new raw decoded string + /// @return new raw decoded wstring std::wstring str2wstr_utf8(const std::string& s); + /// @brief Encode raw u32string to UTF-8 + /// @param ws source raw wstring + /// @return new UTF-8 encoded string + std::string u32str2str_utf8(const std::u32string& ws); + + /// @brief Decode UTF-8 string + /// @param s source encoded string + /// @return new raw decoded u32string + std::u32string str2u32str_utf8(const std::string& s); + /// @brief Calculated length of UTF-8 encoded string that fits into maxSize /// @param s source UTF-8 encoded string view /// @param maxSize max encoded string length after crop diff --git a/test/util/stringutil.cpp b/test/util/stringutil.cpp index ed6e2b00..7bd0f85c 100644 --- a/test/util/stringutil.cpp +++ b/test/util/stringutil.cpp @@ -8,3 +8,10 @@ TEST(stringutil, crop_utf8) { str = str.substr(0, util::crop_utf8(str, 7)); EXPECT_EQ(str, u8"при"); } + +TEST(stringutil, utf8) { + std::string str = u8"テキストデモ"; + auto u32str = util::str2u32str_utf8(str); + std::string str2 = util::u32str2str_utf8(u32str); + EXPECT_EQ(str, str2); +} From 77229b8d9aae249d3a95d7213bdb6326d05755a2 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 15:10:51 +0300 Subject: [PATCH 11/61] add StructMapping::setUnicode --- src/data/StructMapper.cpp | 35 ++++++++++++++----- src/data/StructMapper.hpp | 69 ++++++++++++++++++++++++++++++++++++-- test/data/StructMapper.cpp | 15 +++++++++ 3 files changed, 108 insertions(+), 11 deletions(-) diff --git a/src/data/StructMapper.cpp b/src/data/StructMapper.cpp index 1e26f816..c52e0589 100644 --- a/src/data/StructMapper.cpp +++ b/src/data/StructMapper.cpp @@ -1,10 +1,12 @@ #include "StructMapper.hpp" #include +#include #include #include #include "util/data_io.hpp" +#include "util/stringutil.hpp" using namespace data; @@ -55,7 +57,7 @@ void StructMapping::setInteger( ) { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { - throw std::runtime_error( + throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); } auto ptr = dst + field.offset + index * sizeof_type(field.type); @@ -79,7 +81,7 @@ void StructMapping::setNumber( ) { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { - throw std::runtime_error( + throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); } auto ptr = dst + field.offset + index * sizeof_type(field.type); @@ -103,7 +105,7 @@ void StructMapping::setNumber( } } -void StructMapping::setChars( +size_t StructMapping::setChars( ubyte* dst, std::string_view value, const std::string& name ) { const auto& field = requreField(name); @@ -111,8 +113,23 @@ void StructMapping::setChars( throw std::runtime_error("'char' field type required"); } auto ptr = reinterpret_cast(dst + field.offset); - std::memcpy(ptr, value.data(), - std::min(value.size(), static_cast(field.elements))); + auto size = std::min(value.size(), static_cast(field.elements)); + std::memcpy(ptr, value.data(), size); + return size; +} + +size_t StructMapping::setUnicode( + ubyte* dst, std::string_view value, const std::string& name +) { + const auto& field = requreField(name); + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto text = std::string_view(value.data(), value.size()); + size_t size = util::crop_utf8(text, field.elements); + auto ptr = reinterpret_cast(dst + field.offset); + std::memcpy(ptr, value.data(), size); + return size; } template @@ -125,7 +142,7 @@ integer_t StructMapping::getInteger( ) const { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { - throw std::runtime_error( + throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); } auto ptr = src + field.offset + index * sizeof_type(field.type); @@ -145,7 +162,7 @@ number_t StructMapping::getNumber( ) const { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { - throw std::runtime_error( + throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); } auto ptr = src + field.offset + index * sizeof_type(field.type); @@ -180,6 +197,6 @@ std::string_view StructMapping::getChars( if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); } - auto ptr = src + field.offset; - return std::string_view(reinterpret_cast(ptr), field.elements); + auto ptr = reinterpret_cast(src + field.offset); + return std::string_view(ptr, strnlen(ptr, field.elements)); } diff --git a/src/data/StructMapper.hpp b/src/data/StructMapper.hpp index 358105f9..7f1716b5 100644 --- a/src/data/StructMapper.hpp +++ b/src/data/StructMapper.hpp @@ -44,6 +44,9 @@ namespace data { indices(std::move(indices)) {} + /// @brief Get field by name. Returns nullptr if field not found. + /// @param name field name + /// @return nullable field pointer const Field* getField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { @@ -53,16 +56,78 @@ namespace data { return &fields.at(found->second); } + /// @brief Get field by name + /// @throws std::runtime_exception - field not found + /// @param name field name + /// @return read-only field reference const Field& requreField(const std::string& name) const; + /// @brief Get integer from specified field. + /// Types: (i8, i16, i32, i64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param src source buffer + /// @param name field name + /// @param index array index + /// @return field value + [[nodiscard]] integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const; + + /// @brief Get floating-point number from specified field. + /// Types: (f32, f64, i8, i16, i32, i64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param src source buffer + /// @param name field name + /// @param index array index + /// @return field value + [[nodiscard]] number_t getNumber(const ubyte* src, const std::string& name, int index=0) const; + + /// @brief Get read-only chars array as string_view. + /// @param src source buffer + /// @param name field name + [[nodiscard]] std::string_view getChars(const ubyte* src, const std::string& name) const; + /// @brief Set field integer value. + /// Types: (i8, i16, i32, i64, f32, f64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param dst destination buffer + /// @param value value + /// @param name field name + /// @param index array index void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0); - void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0); - void setChars(ubyte* dst, std::string_view value, const std::string& name); + /// @brief Set field numeric value. + /// Types: (f32, f64) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param dst destination buffer + /// @param value value + /// @param name field name + /// @param index array index + void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0); + + /// @brief Replace chars array to given ASCII string + /// @throws std::runtime_exception - field not found + /// @see StructMapper::setUnicode - utf-8 version of setChars + /// @param dst destination buffer + /// @param value ASCII string + /// @param name field name + /// @return number of written string chars + size_t setChars(ubyte* dst, std::string_view value, const std::string& name); + + /// @brief Unicode-safe version of setChars + /// @throws std::runtime_exception - field not found + /// @param dst destination buffer + /// @param value utf-8 string + /// @param name field name + /// @return number of written string chars + size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name); + + /// @return total structure size (bytes) int size() const { return totalSize; } diff --git a/test/data/StructMapper.cpp b/test/data/StructMapper.cpp index 67aeb97b..78e9204a 100644 --- a/test/data/StructMapper.cpp +++ b/test/data/StructMapper.cpp @@ -24,3 +24,18 @@ TEST(StructMapper, ReadWrite) { mapping.setChars(buffer, "hello", "s"); EXPECT_EQ(mapping.getChars(buffer, "s"), "hell"); } + +TEST(StructMapper, Unicode) { + ubyte buffer[8] {}; + std::vector fields { + Field {FieldType::CHAR, "text", 5}, + }; + auto mapping = StructMapping::create(fields); + EXPECT_EQ(mapping.size(), 5); + + mapping.setUnicode(buffer, u8"テキストデモ", "text"); + EXPECT_EQ(mapping.getChars(buffer, "text"), std::string(u8"テ")); + + mapping.setUnicode(buffer, u8"пример", "text"); + EXPECT_EQ(mapping.getChars(buffer, "text"), std::string(u8"пр")); +} From d0bc6798157b4c3bccf9af29e809711f085e6425 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 17:07:08 +0300 Subject: [PATCH 12/61] update SmallHeap docs --- src/util/SmallHeap.hpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/util/SmallHeap.hpp b/src/util/SmallHeap.hpp index ef590ff0..642ece2f 100644 --- a/src/util/SmallHeap.hpp +++ b/src/util/SmallHeap.hpp @@ -6,6 +6,13 @@ #include namespace util { + // TODO: make it safer (minimize raw temporary pointers use) + /// @brief Simple heap implementation for memory-optimal sparse array of + /// small different structures + /// @note alignment is not impemented + /// (impractical in the context of scripting and memory consumption) + /// @tparam Tindex entry index type + /// @tparam Tsize entry size type template class SmallHeap { std::vector buffer; @@ -13,6 +20,9 @@ namespace util { public: SmallHeap() : entriesCount(0) {} + /// @brief Find current entry address by index + /// @param index entry index + /// @return nullptr if entry does not exists uint8_t* find(Tindex index) { auto data = buffer.data(); for (size_t i = 0; i < entriesCount; i++) { @@ -30,6 +40,8 @@ namespace util { return nullptr; } + /// @brief Erase entry from the heap + /// @param ptr valid entry pointer void free(uint8_t* ptr) { if (ptr == nullptr) { return; @@ -44,6 +56,11 @@ namespace util { entriesCount--; } + /// @brief Create or update entry (size) + /// @param index entry index + /// @param size entry size + /// @return temporary entry pointer + /// (invalid after next allocate(...) or free(...)) uint8_t* allocate(Tindex index, Tsize size) { if (size == 0) { throw std::runtime_error("size is 0"); @@ -84,6 +101,8 @@ namespace util { return data + sizeof(Tsize); } + /// @param ptr valid entry pointer + /// @return entry size Tsize sizeOf(uint8_t* ptr) const { if (ptr == nullptr) { return 0; @@ -91,10 +110,12 @@ namespace util { return *(reinterpret_cast(ptr)-1); } + /// @return number of entries Tindex count() const { return entriesCount; } + /// @return total used bytes including entries metadata size_t size() const { return buffer.size(); } From bcd6f40ddb7497e1bfe0bc164e5852cc5dbbe74e Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 29 Aug 2024 18:56:26 +0300 Subject: [PATCH 13/61] rename StructMapping to StructLayout --- .../{StructMapper.cpp => StructLayout.cpp} | 22 +++++----- .../{StructMapper.hpp => StructLayout.hpp} | 14 ++++--- src/util/SmallHeap.hpp | 5 ++- test/data/StructLayout.cpp | 41 +++++++++++++++++++ test/data/StructMapper.cpp | 41 ------------------- 5 files changed, 63 insertions(+), 60 deletions(-) rename src/data/{StructMapper.cpp => StructLayout.cpp} (93%) rename src/data/{StructMapper.hpp => StructLayout.hpp} (94%) create mode 100644 test/data/StructLayout.cpp delete mode 100644 test/data/StructMapper.cpp diff --git a/src/data/StructMapper.cpp b/src/data/StructLayout.cpp similarity index 93% rename from src/data/StructMapper.cpp rename to src/data/StructLayout.cpp index c52e0589..8fe3682a 100644 --- a/src/data/StructMapper.cpp +++ b/src/data/StructLayout.cpp @@ -1,4 +1,4 @@ -#include "StructMapper.hpp" +#include "StructLayout.hpp" #include #include @@ -13,7 +13,7 @@ using namespace data; static_assert(sizeof(float) == sizeof(int32_t)); static_assert(sizeof(double) == sizeof(int64_t)); -StructMapping StructMapping::create(const std::vector& fields) { +StructLayout StructLayout::create(const std::vector& fields) { std::vector builtFields = fields; std::unordered_map indices; @@ -32,11 +32,11 @@ StructMapping StructMapping::create(const std::vector& fields) { indices[field.name] = i; offset += field.size; } - return StructMapping( + return StructLayout( offset, std::move(builtFields), std::move(indices)); } -const Field& StructMapping::requreField(const std::string& name) const { +const Field& StructLayout::requreField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { throw std::runtime_error("field '"+name+"' does not exist"); @@ -52,7 +52,7 @@ static void set_int(ubyte* dst, integer_t value) { *reinterpret_cast(dst) = out_value; } -void StructMapping::setInteger( +void StructLayout::setInteger( ubyte* dst, integer_t value, const std::string& name, int index ) { const auto& field = requreField(name); @@ -76,7 +76,7 @@ void StructMapping::setInteger( } } -void StructMapping::setNumber( +void StructLayout::setNumber( ubyte* dst, number_t value, const std::string& name, int index ) { const auto& field = requreField(name); @@ -105,7 +105,7 @@ void StructMapping::setNumber( } } -size_t StructMapping::setChars( +size_t StructLayout::setChars( ubyte* dst, std::string_view value, const std::string& name ) { const auto& field = requreField(name); @@ -118,7 +118,7 @@ size_t StructMapping::setChars( return size; } -size_t StructMapping::setUnicode( +size_t StructLayout::setUnicode( ubyte* dst, std::string_view value, const std::string& name ) { const auto& field = requreField(name); @@ -137,7 +137,7 @@ static T get_int(const ubyte* src) { return dataio::le2h(*reinterpret_cast(src)); } -integer_t StructMapping::getInteger( +integer_t StructLayout::getInteger( const ubyte* src, const std::string& name, int index ) const { const auto& field = requreField(name); @@ -157,7 +157,7 @@ integer_t StructMapping::getInteger( } } -number_t StructMapping::getNumber( +number_t StructLayout::getNumber( const ubyte* src, const std::string& name, int index ) const { const auto& field = requreField(name); @@ -190,7 +190,7 @@ number_t StructMapping::getNumber( throw std::runtime_error("type error"); } -std::string_view StructMapping::getChars( +std::string_view StructLayout::getChars( const ubyte* src, const std::string& name ) const { const auto& field = requreField(name); diff --git a/src/data/StructMapper.hpp b/src/data/StructLayout.hpp similarity index 94% rename from src/data/StructMapper.hpp rename to src/data/StructLayout.hpp index 7f1716b5..22d255ab 100644 --- a/src/data/StructMapper.hpp +++ b/src/data/StructLayout.hpp @@ -30,12 +30,12 @@ namespace data { int size; }; - class StructMapping { + class StructLayout { int totalSize; std::vector fields; std::unordered_map indices; public: - StructMapping( + StructLayout( int totalSize, std::vector fields, std::unordered_map indices @@ -47,6 +47,7 @@ namespace data { /// @brief Get field by name. Returns nullptr if field not found. /// @param name field name /// @return nullable field pointer + [[nodiscard]] const Field* getField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { @@ -128,18 +129,19 @@ namespace data { size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name); /// @return total structure size (bytes) - int size() const { + [[nodiscard]] size_t size() const { return totalSize; } - static StructMapping create(const std::vector& fields); + [[nodiscard]] + static StructLayout create(const std::vector& fields); }; class StructAccess { - const StructMapping& mapping; + const StructLayout& mapping; uint8_t* buffer; public: - StructAccess(const StructMapping& mapping, uint8_t* buffer) + StructAccess(const StructLayout& mapping, uint8_t* buffer) : mapping(mapping), buffer(buffer) { } }; diff --git a/src/util/SmallHeap.hpp b/src/util/SmallHeap.hpp index 642ece2f..aea3f51d 100644 --- a/src/util/SmallHeap.hpp +++ b/src/util/SmallHeap.hpp @@ -22,7 +22,8 @@ namespace util { /// @brief Find current entry address by index /// @param index entry index - /// @return nullptr if entry does not exists + /// @return temporary raw pointer or nullptr if entry does not exists + /// @attention pointer becomes invalid after allocate(...) or free(...) uint8_t* find(Tindex index) { auto data = buffer.data(); for (size_t i = 0; i < entriesCount; i++) { @@ -60,7 +61,7 @@ namespace util { /// @param index entry index /// @param size entry size /// @return temporary entry pointer - /// (invalid after next allocate(...) or free(...)) + /// @attention pointer becomes invalid after allocate(...) or free(...) uint8_t* allocate(Tindex index, Tsize size) { if (size == 0) { throw std::runtime_error("size is 0"); diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp new file mode 100644 index 00000000..c948d030 --- /dev/null +++ b/test/data/StructLayout.cpp @@ -0,0 +1,41 @@ +#include "data/StructLayout.hpp" + +#include + +using namespace data; + +TEST(StructLayout, ReadWrite) { + ubyte buffer[16] {}; + + std::vector fields { + Field {FieldType::I8, "a", 1}, + Field {FieldType::CHAR, "s", 4}, + Field {FieldType::F32, "f", 1}, + }; + auto layout = StructLayout::create(fields); + EXPECT_EQ(layout.size(), 9); + + layout.setInteger(buffer, 42, "a"); + EXPECT_EQ(layout.getInteger(buffer, "a"), 42); + + layout.setNumber(buffer, 3.141592f, "f"); + EXPECT_FLOAT_EQ(layout.getNumber(buffer, "f"), 3.141592f); + + layout.setChars(buffer, "hello", "s"); + EXPECT_EQ(layout.getChars(buffer, "s"), "hell"); +} + +TEST(StructLayout, Unicode) { + ubyte buffer[8] {}; + std::vector fields { + Field {FieldType::CHAR, "text", 5}, + }; + auto layout = StructLayout::create(fields); + EXPECT_EQ(layout.size(), 5); + + layout.setUnicode(buffer, u8"テキストデモ", "text"); + EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"テ")); + + layout.setUnicode(buffer, u8"пример", "text"); + EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"пр")); +} diff --git a/test/data/StructMapper.cpp b/test/data/StructMapper.cpp deleted file mode 100644 index 78e9204a..00000000 --- a/test/data/StructMapper.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "data/StructMapper.hpp" - -#include - -using namespace data; - -TEST(StructMapper, ReadWrite) { - ubyte buffer[16] {}; - - std::vector fields { - Field {FieldType::I8, "a", 1}, - Field {FieldType::CHAR, "s", 4}, - Field {FieldType::F32, "f", 1}, - }; - auto mapping = StructMapping::create(fields); - EXPECT_EQ(mapping.size(), 9); - - mapping.setInteger(buffer, 42, "a"); - EXPECT_EQ(mapping.getInteger(buffer, "a"), 42); - - mapping.setNumber(buffer, 3.141592f, "f"); - EXPECT_FLOAT_EQ(mapping.getNumber(buffer, "f"), 3.141592f); - - mapping.setChars(buffer, "hello", "s"); - EXPECT_EQ(mapping.getChars(buffer, "s"), "hell"); -} - -TEST(StructMapper, Unicode) { - ubyte buffer[8] {}; - std::vector fields { - Field {FieldType::CHAR, "text", 5}, - }; - auto mapping = StructMapping::create(fields); - EXPECT_EQ(mapping.size(), 5); - - mapping.setUnicode(buffer, u8"テキストデモ", "text"); - EXPECT_EQ(mapping.getChars(buffer, "text"), std::string(u8"テ")); - - mapping.setUnicode(buffer, u8"пример", "text"); - EXPECT_EQ(mapping.getChars(buffer, "text"), std::string(u8"пр")); -} From 5afbad1bd69b4748d0184de54c3aee2f3fb7c9bc Mon Sep 17 00:00:00 2001 From: MihailRis Date: Fri, 30 Aug 2024 06:13:58 +0300 Subject: [PATCH 14/61] fix skeleton:get_texture --- src/logic/scripting/lua/lib__skeleton.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/logic/scripting/lua/lib__skeleton.cpp b/src/logic/scripting/lua/lib__skeleton.cpp index 9c6295d2..1e0365b7 100644 --- a/src/logic/scripting/lua/lib__skeleton.cpp +++ b/src/logic/scripting/lua/lib__skeleton.cpp @@ -62,8 +62,6 @@ static int l_set_matrix(lua::State* L) { static int l_get_texture(lua::State* L) { if (auto entity = get_entity(L, 1)) { auto& skeleton = entity->getSkeleton(); - skeleton.textures[lua::require_string(L, 2)] = - lua::require_string(L, 3); const auto& found = skeleton.textures.find(lua::require_string(L, 2)); if (found != skeleton.textures.end()) { return lua::pushstring(L, found->second); From caa5e1b03b346dcd65b76627e2d591874182d036 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Fri, 30 Aug 2024 06:16:41 +0300 Subject: [PATCH 15/61] add StructLayout::convert (WIP) --- src/data/StructLayout.cpp | 38 +++++++++++++++++++++++++++++++----- src/data/StructLayout.hpp | 40 +++++++++++++++++++++++++------------- test/data/StructLayout.cpp | 4 ++++ 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 8fe3682a..3cf30a4f 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include "util/data_io.hpp" @@ -36,6 +35,35 @@ StructLayout StructLayout::create(const std::vector& fields) { offset, std::move(builtFields), std::move(indices)); } +static inline constexpr bool is_integer_type(FieldType type) { + return (type >= FieldType::I8 && type <= FieldType::I64) || + type == FieldType::CHAR; +} + +static inline constexpr bool is_floating_point_type(FieldType type) { + return type == FieldType::F32 || type == FieldType::F64; +} + +static inline constexpr bool is_numeric_type(FieldType type) { + return is_floating_point_type(type) || is_integer_type(type); +} + +void StructLayout::convert( + const StructLayout& srcLayout, + const ubyte* src, + ubyte* dst, + bool allowDataLoss +) const { + for (const Field& field : fields) { + auto srcField = srcLayout.getField(field.name); + if (srcField == nullptr) { + std::memset(dst + field.offset, 0, field.size); + continue; + } + // TODO: implement + } +} + const Field& StructLayout::requreField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { @@ -54,7 +82,7 @@ static void set_int(ubyte* dst, integer_t value) { void StructLayout::setInteger( ubyte* dst, integer_t value, const std::string& name, int index -) { +) const { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( @@ -78,7 +106,7 @@ void StructLayout::setInteger( void StructLayout::setNumber( ubyte* dst, number_t value, const std::string& name, int index -) { +) const { const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( @@ -107,7 +135,7 @@ void StructLayout::setNumber( size_t StructLayout::setChars( ubyte* dst, std::string_view value, const std::string& name -) { +) const { const auto& field = requreField(name); if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); @@ -120,7 +148,7 @@ size_t StructLayout::setChars( size_t StructLayout::setUnicode( ubyte* dst, std::string_view value, const std::string& name -) { +) const { const auto& field = requreField(name); if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index 22d255ab..e3f819af 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include "typedefs.hpp" @@ -19,6 +20,11 @@ namespace data { return sizes[static_cast(type)]; } + class dataloss_error : public std::runtime_error { + public: + dataloss_error(const std::string& message) : std::runtime_error(message) {} + }; + struct Field { FieldType type; std::string name; @@ -99,7 +105,7 @@ namespace data { /// @param value value /// @param name field name /// @param index array index - void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0); + void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0) const; /// @brief Set field numeric value. /// Types: (f32, f64) @@ -109,7 +115,7 @@ namespace data { /// @param value value /// @param name field name /// @param index array index - void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0); + void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0) const; /// @brief Replace chars array to given ASCII string /// @throws std::runtime_exception - field not found @@ -118,7 +124,7 @@ namespace data { /// @param value ASCII string /// @param name field name /// @return number of written string chars - size_t setChars(ubyte* dst, std::string_view value, const std::string& name); + size_t setChars(ubyte* dst, std::string_view value, const std::string& name) const; /// @brief Unicode-safe version of setChars /// @throws std::runtime_exception - field not found @@ -126,23 +132,31 @@ namespace data { /// @param value utf-8 string /// @param name field name /// @return number of written string chars - size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name); + size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name) const; /// @return total structure size (bytes) [[nodiscard]] size_t size() const { return totalSize; } + /// @brief Convert structure data from srcLayout to this layout. + /// @param srcLayout source structure layout + /// @param src source data + /// @param dst destination buffer + /// (size must be enough to store converted structure) + /// @param allowDataLoss allow to drop fields that are not present in + /// this layout or have incompatible types + /// @throws data::dataloss_error - data loss detected and allowDataLoss + /// is set to false + void convert( + const StructLayout& srcLayout, + const ubyte* src, + ubyte* dst, + bool allowDataLoss) const; + + /// TODO: add checkCompatibility method + [[nodiscard]] static StructLayout create(const std::vector& fields); }; - - class StructAccess { - const StructLayout& mapping; - uint8_t* buffer; - public: - StructAccess(const StructLayout& mapping, uint8_t* buffer) - : mapping(mapping), buffer(buffer) { - } - }; } diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp index c948d030..ae5ea1d3 100644 --- a/test/data/StructLayout.cpp +++ b/test/data/StructLayout.cpp @@ -39,3 +39,7 @@ TEST(StructLayout, Unicode) { layout.setUnicode(buffer, u8"пример", "text"); EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"пр")); } + +TEST(StructLayout, Convert) { + +} From 8baabf4c0daba3ae138a78cae1fd26bbaca159d5 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 31 Aug 2024 08:47:16 +0300 Subject: [PATCH 16/61] add checkCompatibility method, convertStrategy field, tests --- src/data/StructLayout.cpp | 107 +++++++++++++++++++++++++++++++++++-- src/data/StructLayout.hpp | 27 +++++++++- test/data/StructLayout.cpp | 73 ++++++++++++++++++++++++- 3 files changed, 199 insertions(+), 8 deletions(-) diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 3cf30a4f..0f8154b7 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -1,6 +1,7 @@ #include "StructLayout.hpp" #include +#include #include #include @@ -48,22 +49,118 @@ static inline constexpr bool is_numeric_type(FieldType type) { return is_floating_point_type(type) || is_integer_type(type); } +static inline FieldIncapatibilityType checkIncapatibility( + const Field& srcField, const Field& dstField +) { + auto type = FieldIncapatibilityType::NONE; + if (dstField.elements < srcField.elements) { + type = FieldIncapatibilityType::DATA_LOSS; + } + if (srcField.type == dstField.type) { + return type; + } + if (is_numeric_type(srcField.type) && is_numeric_type(dstField.type)) { + int sizediff = + sizeof_type(dstField.type) - sizeof_type(srcField.type); + if (sizediff < 0) { + type = std::max(type, FieldIncapatibilityType::DATA_LOSS); + } + } else { + type = std::max(type, FieldIncapatibilityType::TYPE_ERROR); + } + return type; +} + +static inline integer_t clamp_value(integer_t value, FieldType type) { + auto typesize = sizeof_type(type) * CHAR_BIT; + integer_t minval = -(1 << (typesize-1)); + integer_t maxval = (1 << (typesize-1))-1; + return std::min(maxval, std::max(minval, value)); +} + +static void reset_integer( + const StructLayout& srcLayout, + const StructLayout& dstLayout, + const Field& field, + const Field& dstField, + const ubyte* src, + ubyte* dst +) { + int elements = std::min(field.elements, dstField.elements); + for (int i = 0; i < elements; i++) { + auto value = srcLayout.getInteger(src, field.name, i); + auto clamped = clamp_value(value, dstField.type); + if (dstField.convertStrategy == FieldConvertStrategy::CLAMP) { + value = clamped; + } else { + if (clamped != value) { + value = 0; + } + } + dstLayout.setInteger(dst, value, field.name, i); + } +} + +static void reset_number( + const StructLayout& srcLayout, + const StructLayout& dstLayout, + const Field& field, + const Field& dstField, + const ubyte* src, + ubyte* dst +) { + int elements = std::min(field.elements, dstField.elements); + for (int i = 0; i < elements; i++) { + auto value = srcLayout.getNumber(src, field.name, i); + dstLayout.setNumber(dst, value, field.name, i); + } +} + void StructLayout::convert( const StructLayout& srcLayout, const ubyte* src, ubyte* dst, bool allowDataLoss ) const { - for (const Field& field : fields) { - auto srcField = srcLayout.getField(field.name); - if (srcField == nullptr) { - std::memset(dst + field.offset, 0, field.size); + std::memset(dst, 0, totalSize); + for (const Field& field : srcLayout.fields) { + auto dstField = getField(field.name); + if (dstField == nullptr) { continue; } - // TODO: implement + auto type = checkIncapatibility(field, *dstField); + if (type == FieldIncapatibilityType::TYPE_ERROR) { + continue; + } + // can't just memcpy, because field type may be changed without data loss + if (is_integer_type(field.type) || + (is_floating_point_type(field.type) && + is_integer_type(dstField->type))) { + reset_integer(srcLayout, *this, field, *dstField, src, dst); + } else if (is_floating_point_type(dstField->type)) { + reset_number(srcLayout, *this, field, *dstField, src, dst); + } } } +std::vector StructLayout::checkCompatibility( + const StructLayout& dstLayout +) { + std::vector report; + for (const Field& field : fields) { + auto dstField = dstLayout.getField(field.name); + if (dstField == nullptr) { + report.push_back({field.name, FieldIncapatibilityType::MISSING}); + continue; + } + auto type = checkIncapatibility(field, *dstField); + if (type != FieldIncapatibilityType::NONE) { + report.push_back({field.name, type}); + } + } + return report; +} + const Field& StructLayout::requreField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index e3f819af..0cb07605 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "typedefs.hpp" @@ -13,6 +14,19 @@ namespace data { COUNT }; + /// @brief Sorted by severity + enum class FieldIncapatibilityType { + NONE = 0, + DATA_LOSS, + TYPE_ERROR, + MISSING, + }; + + struct FieldIncapatibility { + std::string name; + FieldIncapatibilityType type; + }; + inline constexpr int sizeof_type(FieldType type) { const int sizes[static_cast(FieldType::COUNT)] = { 1, 2, 4, 8, 4, 8, 1 @@ -25,11 +39,21 @@ namespace data { dataloss_error(const std::string& message) : std::runtime_error(message) {} }; + /// @brief Strategy will be used if value can't be left the same on conversion + enum class FieldConvertStrategy { + /// @brief Reset field value + RESET, + /// @brief Clamp field value if out of type range + CLAMP + }; + struct Field { FieldType type; std::string name; /// @brief Number of field elements (array size) int elements; + /// @brief Strategy will be used in data loss case + FieldConvertStrategy convertStrategy; /// @brief Byte offset of the field int offset; /// @brief Byte size of the field @@ -154,7 +178,8 @@ namespace data { ubyte* dst, bool allowDataLoss) const; - /// TODO: add checkCompatibility method + std::vector checkCompatibility( + const StructLayout& dstLayout); [[nodiscard]] static StructLayout create(const std::vector& fields); diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp index ae5ea1d3..97738bcc 100644 --- a/test/data/StructLayout.cpp +++ b/test/data/StructLayout.cpp @@ -1,6 +1,8 @@ #include "data/StructLayout.hpp" #include +#include +#include using namespace data; @@ -40,6 +42,73 @@ TEST(StructLayout, Unicode) { EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"пр")); } -TEST(StructLayout, Convert) { - +TEST(StructLayout, ConvertReorder) { + ubyte src[16] {}; + std::vector srcFields { + Field {FieldType::CHAR, "text", 5}, + Field {FieldType::F64, "pi", 1}, + }; + auto srcLayout = StructLayout::create(srcFields); + srcLayout.setChars(src, "truth", "text"); + srcLayout.setNumber(src, 3.141592, "pi"); + + EXPECT_EQ(srcLayout.getChars(src, "text"), "truth"); + EXPECT_DOUBLE_EQ(srcLayout.getNumber(src, "pi"), 3.141592); + + ubyte dst[32] {}; + std::vector dstFields { + Field {FieldType::I64, "arr", 2}, // an array of 2 i64 + Field {FieldType::CHAR, "text", 5}, + Field {FieldType::F64, "pi", 1}, + }; + auto dstLayout = StructLayout::create(dstFields); + EXPECT_EQ(srcLayout.checkCompatibility(dstLayout).size(), 0); + dstLayout.convert(srcLayout, src, dst, false); + + EXPECT_EQ(dstLayout.getChars(dst, "text"), "truth"); + EXPECT_DOUBLE_EQ(dstLayout.getNumber(dst, "pi"), 3.141592); +} + +TEST(StructLayout, ConvertWithLoss) { + ubyte src[32] {}; + std::vector srcFields { + Field {FieldType::CHAR, "text", 5}, + Field {FieldType::I16, "someint", 1}, + Field {FieldType::F64, "pi", 1}, + }; + auto srcLayout = StructLayout::create(srcFields); + srcLayout.setChars(src, "truth", "text"); + srcLayout.setInteger(src, 150, "someint"); + srcLayout.setNumber(src, 3.141592, "pi"); + + EXPECT_EQ(srcLayout.getChars(src, "text"), "truth"); + EXPECT_EQ(srcLayout.getInteger(src, "someint"), 150); + EXPECT_DOUBLE_EQ(srcLayout.getNumber(src, "pi"), 3.141592); + + ubyte dst[8] {}; + std::vector dstFields { + Field {FieldType::CHAR, "text", 3}, + Field {FieldType::I8, "someint", 1, FieldConvertStrategy::CLAMP}, + }; + auto dstLayout = StructLayout::create(dstFields); + auto report = srcLayout.checkCompatibility(dstLayout); + std::sort(report.begin(), report.end(), [](const auto& a, const auto& b) { + return a.name < b.name; + }); + EXPECT_EQ(report.size(), 3); + + EXPECT_EQ(report[0].name, "pi"); + EXPECT_EQ(report[0].type, FieldIncapatibilityType::MISSING); + + EXPECT_EQ(report[1].name, "someint"); + EXPECT_EQ(report[1].type, FieldIncapatibilityType::DATA_LOSS); + + EXPECT_EQ(report[2].name, "text"); + EXPECT_EQ(report[2].type, FieldIncapatibilityType::DATA_LOSS); + + // convert with losses + dstLayout.convert(srcLayout, src, dst, true); + + EXPECT_EQ(dstLayout.getChars(dst, "text"), "tru"); + EXPECT_EQ(dstLayout.getInteger(dst, "someint"), INT8_MAX); } From 35079d1b49c6cf8ce89f0e406fab6501cfe10bfe Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 31 Aug 2024 09:25:05 +0300 Subject: [PATCH 17/61] rename setChars to setAscii --- src/data/StructLayout.cpp | 2 +- src/data/StructLayout.hpp | 6 +++--- test/data/StructLayout.cpp | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 0f8154b7..417bc6e7 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -230,7 +230,7 @@ void StructLayout::setNumber( } } -size_t StructLayout::setChars( +size_t StructLayout::setAscii( ubyte* dst, std::string_view value, const std::string& name ) const { const auto& field = requreField(name); diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index 0cb07605..b30bcea6 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -143,14 +143,14 @@ namespace data { /// @brief Replace chars array to given ASCII string /// @throws std::runtime_exception - field not found - /// @see StructMapper::setUnicode - utf-8 version of setChars + /// @see StructMapper::setUnicode - utf-8 version of setAscii /// @param dst destination buffer /// @param value ASCII string /// @param name field name /// @return number of written string chars - size_t setChars(ubyte* dst, std::string_view value, const std::string& name) const; + size_t setAscii(ubyte* dst, std::string_view value, const std::string& name) const; - /// @brief Unicode-safe version of setChars + /// @brief Unicode-safe version of setAscii /// @throws std::runtime_exception - field not found /// @param dst destination buffer /// @param value utf-8 string diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp index 97738bcc..720434ef 100644 --- a/test/data/StructLayout.cpp +++ b/test/data/StructLayout.cpp @@ -23,7 +23,7 @@ TEST(StructLayout, ReadWrite) { layout.setNumber(buffer, 3.141592f, "f"); EXPECT_FLOAT_EQ(layout.getNumber(buffer, "f"), 3.141592f); - layout.setChars(buffer, "hello", "s"); + layout.setAscii(buffer, "hello", "s"); EXPECT_EQ(layout.getChars(buffer, "s"), "hell"); } @@ -49,7 +49,7 @@ TEST(StructLayout, ConvertReorder) { Field {FieldType::F64, "pi", 1}, }; auto srcLayout = StructLayout::create(srcFields); - srcLayout.setChars(src, "truth", "text"); + srcLayout.setAscii(src, "truth", "text"); srcLayout.setNumber(src, 3.141592, "pi"); EXPECT_EQ(srcLayout.getChars(src, "text"), "truth"); @@ -77,7 +77,7 @@ TEST(StructLayout, ConvertWithLoss) { Field {FieldType::F64, "pi", 1}, }; auto srcLayout = StructLayout::create(srcFields); - srcLayout.setChars(src, "truth", "text"); + srcLayout.setAscii(src, "truth", "text"); srcLayout.setInteger(src, 150, "someint"); srcLayout.setNumber(src, 3.141592, "pi"); From dfb5baf79b1e00b164088ec803b5a436c6eaa56c Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 31 Aug 2024 19:58:41 +0300 Subject: [PATCH 18/61] fix StructLayout::getNumber --- src/data/StructLayout.cpp | 3 ++- test/data/StructLayout.cpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 417bc6e7..01a89d2d 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -310,7 +310,8 @@ number_t StructLayout::getNumber( case FieldType::I64: case FieldType::CHAR: return getInteger(src, name, index); - + default: + throw std::runtime_error("type error"); } throw std::runtime_error("type error"); } diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp index 720434ef..bf6f8e1d 100644 --- a/test/data/StructLayout.cpp +++ b/test/data/StructLayout.cpp @@ -92,6 +92,8 @@ TEST(StructLayout, ConvertWithLoss) { }; auto dstLayout = StructLayout::create(dstFields); auto report = srcLayout.checkCompatibility(dstLayout); + + // check report std::sort(report.begin(), report.end(), [](const auto& a, const auto& b) { return a.name < b.name; }); From 0f53d5b83546b5683791328a34772f8e5fc4df14 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 1 Sep 2024 19:44:24 +0300 Subject: [PATCH 19/61] refactor WorldRegions --- src/data/StructLayout.cpp | 1 - src/files/RegionsLayer.cpp | 229 ++++++++++++++++++++++++++++ src/files/WorldFiles.cpp | 1 - src/files/WorldFiles.hpp | 4 +- src/files/WorldRegions.cpp | 282 ++++------------------------------- src/files/WorldRegions.hpp | 108 +++++++++----- src/voxels/ChunksStorage.cpp | 2 +- 7 files changed, 326 insertions(+), 301 deletions(-) create mode 100644 src/files/RegionsLayer.cpp diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 01a89d2d..9124b68a 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -313,7 +313,6 @@ number_t StructLayout::getNumber( default: throw std::runtime_error("type error"); } - throw std::runtime_error("type error"); } std::string_view StructLayout::getChars( diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp new file mode 100644 index 00000000..2fc405de --- /dev/null +++ b/src/files/RegionsLayer.cpp @@ -0,0 +1,229 @@ +#include "WorldRegions.hpp" + +#include + +#include "util/data_io.hpp" + +#define REGION_FORMAT_MAGIC ".VOXREG" + +static fs::path get_region_filename(int x, int z) { + return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); +} + +/// @brief Read missing chunks data (null pointers) from region file +static void fetchChunks(WorldRegion* region, int x, int z, regfile* file) { + auto* chunks = region->getChunks(); + uint32_t* sizes = region->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; + int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; + if (chunks[i] == nullptr) { + chunks[i] = + RegionsLayer::readChunkData(chunk_x, chunk_z, sizes[i], file); + } + } +} + +regfile::regfile(fs::path filename) : file(std::move(filename)) { + if (file.length() < REGION_HEADER_SIZE) + throw std::runtime_error("incomplete region file header"); + char header[REGION_HEADER_SIZE]; + file.read(header, REGION_HEADER_SIZE); + + // avoid of use strcmp_s + if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) != + REGION_FORMAT_MAGIC) { + throw std::runtime_error("invalid region file magic number"); + } + version = header[8]; + if (static_cast(version) > REGION_FORMAT_VERSION) { + throw illegal_region_format( + "region format " + std::to_string(version) + " is not supported" + ); + } +} + +std::unique_ptr regfile::read(int index, uint32_t& length) { + size_t file_size = file.length(); + size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; + + uint32_t offset; + file.seekg(table_offset + index * 4); + file.read(reinterpret_cast(&offset), 4); + offset = dataio::read_int32_big(reinterpret_cast(&offset), 0); + if (offset == 0) { + return nullptr; + } + + file.seekg(offset); + file.read(reinterpret_cast(&offset), 4); + length = dataio::read_int32_big(reinterpret_cast(&offset), 0); + auto data = std::make_unique(length); + file.read(reinterpret_cast(data.get()), length); + return data; +} + + +void RegionsLayer::closeRegFile(glm::ivec2 coord) { + openRegFiles.erase(coord); + regFilesCv.notify_one(); +} + +regfile_ptr RegionsLayer::useRegFile(glm::ivec2 coord) { + auto* file = openRegFiles[coord].get(); + file->inUse = true; + return regfile_ptr(file, ®FilesCv); +} + +// Marks regfile as used and unmarks when shared_ptr dies +regfile_ptr RegionsLayer::getRegFile(glm::ivec2 coord, bool create) { + { + std::lock_guard lock(regFilesMutex); + const auto found = openRegFiles.find(coord); + if (found != openRegFiles.end()) { + if (found->second->inUse) { + throw std::runtime_error("regfile is currently in use"); + } + return useRegFile(found->first); + } + } + if (create) { + return createRegFile(coord); + } + return nullptr; +} + +regfile_ptr RegionsLayer::createRegFile(glm::ivec2 coord) { + auto file = folder / get_region_filename(coord[0], coord[1]); + if (!fs::exists(file)) { + return nullptr; + } + if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { + std::unique_lock lock(regFilesMutex); + while (true) { + bool closed = false; + // FIXME: bad choosing algorithm + for (auto& entry : openRegFiles) { + if (!entry.second->inUse) { + closeRegFile(entry.first); + closed = true; + break; + } + } + if (closed) { + break; + } + // notified when any regfile gets out of use or closed + regFilesCv.wait(lock); + } + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } else { + std::lock_guard lock(regFilesMutex); + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } +} + +WorldRegion* RegionsLayer::getRegion(int x, int z) { + std::lock_guard lock(mapMutex); + auto found = regions.find({x, z}); + if (found == regions.end()) { + return nullptr; + } + return found->second.get(); +} + +WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) { + if (auto region = getRegion(x, z)) { + return region; + } + std::lock_guard lock(mapMutex); + auto region_ptr = std::make_unique(); + auto region = region_ptr.get(); + regions[{x, z}] = std::move(region_ptr); + return region; +} + +ubyte* RegionsLayer::getData(int x, int z, uint32_t& size) { + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + + WorldRegion* region = getOrCreateRegion(regionX, regionZ); + ubyte* data = region->getChunkData(localX, localZ); + if (data == nullptr) { + auto regfile = getRegFile({regionX, regionZ}); + if (regfile != nullptr) { + auto dataptr = readChunkData(x, z, size, regfile.get()); + if (dataptr) { + data = dataptr.get(); + region->put(localX, localZ, std::move(dataptr), size); + } + } + } + if (data != nullptr) { + size = region->getChunkDataSize(localX, localZ); + return data; + } + return nullptr; +} + +void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) { + fs::path filename = folder / get_region_filename(x, z); + + glm::ivec2 regcoord(x, z); + if (auto regfile = getRegFile(regcoord, false)) { + fetchChunks(entry, x, z, regfile.get()); + + std::lock_guard lock(regFilesMutex); + regfile.reset(); + closeRegFile(regcoord); + } + + char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; + header[8] = REGION_FORMAT_VERSION; + header[9] = 0; // flags + std::ofstream file(filename, std::ios::out | std::ios::binary); + file.write(header, REGION_HEADER_SIZE); + + size_t offset = REGION_HEADER_SIZE; + char intbuf[4] {}; + uint offsets[REGION_CHUNKS_COUNT] {}; + + auto* region = entry->getChunks(); + uint32_t* sizes = entry->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + ubyte* chunk = region[i].get(); + if (chunk == nullptr) { + offsets[i] = 0; + } else { + offsets[i] = offset; + + size_t compressedSize = sizes[i]; + dataio::write_int32_big( + compressedSize, reinterpret_cast(intbuf), 0 + ); + offset += 4 + compressedSize; + + file.write(intbuf, 4); + file.write(reinterpret_cast(chunk), compressedSize); + } + } + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + dataio::write_int32_big( + offsets[i], reinterpret_cast(intbuf), 0 + ); + file.write(intbuf, 4); + } +} + +std::unique_ptr RegionsLayer::readChunkData( + int x, int z, uint32_t& length, regfile* rfile +) { + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + int chunkIndex = localZ * REGION_SIZE + localX; + return rfile->read(chunkIndex, length); +} diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 562c40c0..33ebaf37 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -84,7 +84,6 @@ void WorldFiles::write(const World* world, const Content* content) { if (generatorTestMode) { return; } - writeIndices(content->getIndices()); regions.write(); } diff --git a/src/files/WorldFiles.hpp b/src/files/WorldFiles.hpp index e0e78920..cb9c4b45 100644 --- a/src/files/WorldFiles.hpp +++ b/src/files/WorldFiles.hpp @@ -63,8 +63,6 @@ public: /// @return world folder fs::path getFolder() const; - static const inline std::string WORLD_FILE = "world.json"; - WorldRegions& getRegions() { return regions; } @@ -72,4 +70,6 @@ public: bool doesWriteLights() const { return doWriteLights; } + + static const inline std::string WORLD_FILE = "world.json"; }; diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index d47d64c9..9074c69e 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -11,47 +11,6 @@ #include "maths/voxmaths.hpp" #include "util/data_io.hpp" -#define REGION_FORMAT_MAGIC ".VOXREG" - -regfile::regfile(fs::path filename) : file(std::move(filename)) { - if (file.length() < REGION_HEADER_SIZE) - throw std::runtime_error("incomplete region file header"); - char header[REGION_HEADER_SIZE]; - file.read(header, REGION_HEADER_SIZE); - - // avoid of use strcmp_s - if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) != - REGION_FORMAT_MAGIC) { - throw std::runtime_error("invalid region file magic number"); - } - version = header[8]; - if (static_cast(version) > REGION_FORMAT_VERSION) { - throw illegal_region_format( - "region format " + std::to_string(version) + " is not supported" - ); - } -} - -std::unique_ptr regfile::read(int index, uint32_t& length) { - size_t file_size = file.length(); - size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; - - uint32_t offset; - file.seekg(table_offset + index * 4); - file.read(reinterpret_cast(&offset), 4); - offset = dataio::read_int32_big(reinterpret_cast(&offset), 0); - if (offset == 0) { - return nullptr; - } - - file.seekg(offset); - file.read(reinterpret_cast(&offset), 4); - length = dataio::read_int32_big(reinterpret_cast(&offset), 0); - auto data = std::make_unique(length); - file.read(reinterpret_cast(data.get()), length); - return data; -} - WorldRegion::WorldRegion() : chunksData( std::make_unique[]>(REGION_CHUNKS_COUNT) @@ -76,9 +35,11 @@ uint32_t* WorldRegion::getSizes() const { return sizes.get(); } -void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) { +void WorldRegion::put( + uint x, uint z, std::unique_ptr data, uint32_t size +) { size_t chunk_index = z * REGION_SIZE + x; - chunksData[chunk_index].reset(data); + chunksData[chunk_index] = std::move(data); sizes[chunk_index] = size; } @@ -103,28 +64,6 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { WorldRegions::~WorldRegions() = default; -WorldRegion* WorldRegions::getRegion(int x, int z, int layer) { - RegionsLayer& regions = layers[layer]; - std::lock_guard lock(regions.mutex); - auto found = regions.regions.find(glm::ivec2(x, z)); - if (found == regions.regions.end()) { - return nullptr; - } - return found->second.get(); -} - -WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) { - if (auto region = getRegion(x, z, layer)) { - return region; - } - RegionsLayer& regions = layers[layer]; - std::lock_guard lock(regions.mutex); - auto region_ptr = std::make_unique(); - auto region = region_ptr.get(); - regions.regions[{x, z}] = std::move(region_ptr); - return region; -} - std::unique_ptr WorldRegions::compress( const ubyte* src, size_t srclen, size_t& len ) { @@ -147,189 +86,14 @@ std::unique_ptr WorldRegions::decompress( return decompressed; } -inline void calc_reg_coords( - int x, int z, int& regionX, int& regionZ, int& localX, int& localZ -) { - regionX = floordiv(x, REGION_SIZE); - regionZ = floordiv(z, REGION_SIZE); - localX = x - (regionX * REGION_SIZE); - localZ = z - (regionZ * REGION_SIZE); -} - -std::unique_ptr WorldRegions::readChunkData( - int x, int z, uint32_t& length, regfile* rfile -) { - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - int chunkIndex = localZ * REGION_SIZE + localX; - return rfile->read(chunkIndex, length); -} - -/// @brief Read missing chunks data (null pointers) from region file -void WorldRegions::fetchChunks( - WorldRegion* region, int x, int z, regfile* file -) { - auto* chunks = region->getChunks(); - uint32_t* sizes = region->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; - int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; - if (chunks[i] == nullptr) { - chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file); - } - } -} - -ubyte* WorldRegions::getData(int x, int z, int layer, uint32_t& size) { - if (generatorTestMode) { - return nullptr; - } - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); - ubyte* data = region->getChunkData(localX, localZ); - if (data == nullptr) { - auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer)); - if (regfile != nullptr) { - data = readChunkData(x, z, size, regfile.get()).release(); - } - if (data != nullptr) { - region->put(localX, localZ, data, size); - } - } - if (data != nullptr) { - size = region->getChunkDataSize(localX, localZ); - return data; - } - return nullptr; -} - -regfile_ptr WorldRegions::useRegFile(glm::ivec3 coord) { - auto* file = openRegFiles[coord].get(); - file->inUse = true; - return regfile_ptr(file, ®FilesCv); -} - -void WorldRegions::closeRegFile(glm::ivec3 coord) { - openRegFiles.erase(coord); - regFilesCv.notify_one(); -} - -// Marks regfile as used and unmarks when shared_ptr dies -regfile_ptr WorldRegions::getRegFile(glm::ivec3 coord, bool create) { - { - std::lock_guard lock(regFilesMutex); - const auto found = openRegFiles.find(coord); - if (found != openRegFiles.end()) { - if (found->second->inUse) { - throw std::runtime_error("regfile is currently in use"); - } - return useRegFile(found->first); - } - } - if (create) { - return createRegFile(coord); - } - return nullptr; -} - -regfile_ptr WorldRegions::createRegFile(glm::ivec3 coord) { - fs::path file = - layers[coord[2]].folder / getRegionFilename(coord[0], coord[1]); - if (!fs::exists(file)) { - return nullptr; - } - if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { - std::unique_lock lock(regFilesMutex); - while (true) { - bool closed = false; - // FIXME: bad choosing algorithm - for (auto& entry : openRegFiles) { - if (!entry.second->inUse) { - closeRegFile(entry.first); - closed = true; - break; - } - } - if (closed) { - break; - } - // notified when any regfile gets out of use or closed - regFilesCv.wait(lock); - } - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } else { - std::lock_guard lock(regFilesMutex); - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } -} - -fs::path WorldRegions::getRegionFilename(int x, int z) const { - return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); -} - -void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry) { - fs::path filename = layers[layer].folder / getRegionFilename(x, z); - - glm::ivec3 regcoord(x, z, layer); - if (auto regfile = getRegFile(regcoord, false)) { - fetchChunks(entry, x, z, regfile.get()); - - std::lock_guard lock(regFilesMutex); - regfile.reset(); - closeRegFile(regcoord); - } - - char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; - header[8] = REGION_FORMAT_VERSION; - header[9] = 0; // flags - std::ofstream file(filename, std::ios::out | std::ios::binary); - file.write(header, REGION_HEADER_SIZE); - - size_t offset = REGION_HEADER_SIZE; - char intbuf[4] {}; - uint offsets[REGION_CHUNKS_COUNT] {}; - - auto* region = entry->getChunks(); - uint32_t* sizes = entry->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - ubyte* chunk = region[i].get(); - if (chunk == nullptr) { - offsets[i] = 0; - } else { - offsets[i] = offset; - - size_t compressedSize = sizes[i]; - dataio::write_int32_big( - compressedSize, reinterpret_cast(intbuf), 0 - ); - offset += 4 + compressedSize; - - file.write(intbuf, 4); - file.write(reinterpret_cast(chunk), compressedSize); - } - } - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - dataio::write_int32_big( - offsets[i], reinterpret_cast(intbuf), 0 - ); - file.write(intbuf, 4); - } -} - -void WorldRegions::writeRegions(int layer) { - for (auto& it : layers[layer].regions) { +void RegionsLayer::writeAll() { + for (auto& it : regions) { WorldRegion* region = it.second.get(); if (region->getChunks() == nullptr || !region->isUnsaved()) { continue; } - glm::ivec2 key = it.first; - writeRegion(key[0], key[1], layer, region); + const auto& key = it.first; + writeRegion(key[0], key[1], region); } } @@ -350,9 +114,9 @@ void WorldRegions::put( int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); + WorldRegion* region = layers[layer].getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); - region->put(localX, localZ, data.release(), size); + region->put(localX, localZ, std::move(data), size); } static std::unique_ptr write_inventories( @@ -431,9 +195,9 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { } } -std::unique_ptr WorldRegions::getChunk(int x, int z) { +std::unique_ptr WorldRegions::getVoxels(int x, int z) { uint32_t size; - auto* data = getData(x, z, REGION_LAYER_VOXELS, size); + auto* data = layers[REGION_LAYER_VOXELS].getData(x, z, size); if (data == nullptr) { return nullptr; } @@ -444,7 +208,7 @@ std::unique_ptr WorldRegions::getChunk(int x, int z) { /// @return lights data or nullptr std::unique_ptr WorldRegions::getLights(int x, int z) { uint32_t size; - auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size); + auto* bytes = layers[REGION_LAYER_LIGHTS].getData(x, z, size); if (bytes == nullptr) { return nullptr; } @@ -455,11 +219,11 @@ std::unique_ptr WorldRegions::getLights(int x, int z) { chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { chunk_inventories_map meta; uint32_t bytesSize; - const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize); - if (data == nullptr) { + auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize); + if (bytes == nullptr) { return meta; } - ByteReader reader(data, bytesSize); + ByteReader reader(bytes, bytesSize); auto count = reader.getInt32(); for (int i = 0; i < count; i++) { uint index = reader.getInt32(); @@ -474,8 +238,11 @@ chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { } dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { + if (generatorTestMode) { + return nullptr; + } uint32_t bytesSize; - const ubyte* data = getData(x, z, REGION_LAYER_ENTITIES, bytesSize); + const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize); if (data == nullptr) { return nullptr; } @@ -487,10 +254,10 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { } void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - if (getRegion(x, z, REGION_LAYER_VOXELS)) { + if (layers[REGION_LAYER_VOXELS].getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } - auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS)); + auto regfile = layers[REGION_LAYER_VOXELS].getRegFile({x, z}); if (regfile == nullptr) { throw std::runtime_error("could not open region file"); } @@ -499,7 +266,8 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { int gx = cx + x * REGION_SIZE; int gz = cz + z * REGION_SIZE; uint32_t length; - auto data = readChunkData(gx, gz, length, regfile.get()); + auto data = + RegionsLayer::readChunkData(gx, gz, length, regfile.get()); if (data == nullptr) { continue; } @@ -523,7 +291,7 @@ fs::path WorldRegions::getRegionsFolder(int layer) const { void WorldRegions::write() { for (auto& layer : layers) { fs::create_directories(layer.folder); - writeRegions(layer.layer); + layer.writeAll(); } } diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index e346dddc..e467b798 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -12,6 +12,7 @@ #include "typedefs.hpp" #include "util/BufferPool.hpp" #include "voxels/Chunk.hpp" +#include "maths/voxmaths.hpp" #include "files.hpp" #define GLM_ENABLE_EXPERIMENTAL @@ -30,7 +31,7 @@ inline constexpr uint REGION_SIZE_BIT = 5; inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); inline constexpr uint REGION_FORMAT_VERSION = 2; -inline constexpr uint MAX_OPEN_REGION_FILES = 16; +inline constexpr uint MAX_OPEN_REGION_FILES = 32; class illegal_region_format : public std::runtime_error { public: @@ -47,7 +48,7 @@ public: WorldRegion(); ~WorldRegion(); - void put(uint x, uint z, ubyte* data, uint32_t size); + void put(uint x, uint z, std::unique_ptr data, uint32_t size); ubyte* getChunkData(uint x, uint z); uint getChunkDataSize(uint x, uint z); @@ -72,13 +73,6 @@ struct regfile { using regionsmap = std::unordered_map>; using regionproc = std::function; -struct RegionsLayer { - int layer; - fs::path folder; - regionsmap regions; - std::mutex mutex; -}; - class regfile_ptr { regfile* file; std::condition_variable* cv; @@ -116,18 +110,77 @@ public: } }; -class WorldRegions { - fs::path directory; - std::unordered_map> openRegFiles; +inline void calc_reg_coords( + int x, int z, int& regionX, int& regionZ, int& localX, int& localZ +) { + regionX = floordiv(x, REGION_SIZE); + regionZ = floordiv(z, REGION_SIZE); + localX = x - (regionX * REGION_SIZE); + localZ = z - (regionZ * REGION_SIZE); +} + +struct RegionsLayer { + /// @brief Layer index + int layer; + + /// @brief Regions layer folder + fs::path folder; + + /// @brief In-memory regions data + regionsmap regions; + + /// @brief In-memory regions map mutex + std::mutex mapMutex; + + /// @brief Open region files map + std::unordered_map> openRegFiles; + + /// @brief Open region files map mutex std::mutex regFilesMutex; std::condition_variable regFilesCv; + + regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); + [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord); + regfile_ptr createRegFile(glm::ivec2 coord); + void closeRegFile(glm::ivec2 coord); + + WorldRegion* getRegion(int x, int z); + WorldRegion* getOrCreateRegion(int x, int z); + + /// @brief Get chunk data. Read from file if not loaded yet. + /// @param x chunk x coord + /// @param z chunk z coord + /// @param size [out] chunk data length + /// @return nullptr if no saved chunk data found + [[nodiscard]] ubyte* getData(int x, int z, uint32_t& size); + + /// @brief Write or rewrite region file + /// @param x region X + /// @param z region Z + void writeRegion(int x, int y, WorldRegion* entry); + + /// @brief Write all unsaved regions to files + void writeAll(); + + /// @brief Read chunk data from region file + /// @param x chunk x coord + /// @param z chunk z coord + /// @param length [out] chunk data length + /// @param rfile region file + /// @return nullptr if chunk is not present in region file + [[nodiscard]] static std::unique_ptr readChunkData( + int x, int z, uint32_t& length, regfile* rfile + ); +}; + +class WorldRegions { + /// @brief World directory + fs::path directory; + RegionsLayer layers[4] {}; util::BufferPool bufferPool { std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2}; - WorldRegion* getRegion(int x, int z, int layer); - WorldRegion* getOrCreateRegion(int x, int z, int layer); - /// @brief Compress buffer with extrle /// @param src source buffer /// @param srclen length of the source buffer @@ -145,29 +198,6 @@ class WorldRegions { std::unique_ptr decompress( const ubyte* src, size_t srclen, size_t dstlen ); - - std::unique_ptr readChunkData( - int x, int y, uint32_t& length, regfile* file - ); - - void fetchChunks(WorldRegion* region, int x, int y, regfile* file); - - ubyte* getData(int x, int z, int layer, uint32_t& size); - - regfile_ptr getRegFile(glm::ivec3 coord, bool create = true); - void closeRegFile(glm::ivec3 coord); - regfile_ptr useRegFile(glm::ivec3 coord); - regfile_ptr createRegFile(glm::ivec3 coord); - - fs::path getRegionFilename(int x, int y) const; - - void writeRegions(int layer); - - /// @brief Write or rewrite region file - /// @param x region X - /// @param z region Z - /// @param layer regions layer - void writeRegion(int x, int y, int layer, WorldRegion* entry); public: bool generatorTestMode = false; bool doWriteLights = true; @@ -195,7 +225,7 @@ public: bool rle ); - std::unique_ptr getChunk(int x, int z); + std::unique_ptr getVoxels(int x, int z); std::unique_ptr getLights(int x, int z); chunk_inventories_map fetchInventories(int x, int z); dynamic::Map_sptr fetchEntities(int x, int z); diff --git a/src/voxels/ChunksStorage.cpp b/src/voxels/ChunksStorage.cpp index 9bc0a4f6..23665b61 100644 --- a/src/voxels/ChunksStorage.cpp +++ b/src/voxels/ChunksStorage.cpp @@ -59,7 +59,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { auto chunk = std::make_shared(x, z); store(chunk); - auto data = regions.getChunk(chunk->x, chunk->z); + auto data = regions.getVoxels(chunk->x, chunk->z); if (data) { chunk->decode(data.get()); From 7d193941a435de1252c71a4fa1a728d6ef3604f5 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 1 Sep 2024 21:58:33 +0300 Subject: [PATCH 20/61] add /coders/compression module --- src/coders/compression.cpp | 87 +++++++++++++++++++++++++++++++++++ src/coders/compression.hpp | 30 ++++++++++++ src/files/RegionsLayer.cpp | 4 +- src/files/WorldRegions.cpp | 88 ++++++++++++++---------------------- src/files/WorldRegions.hpp | 36 +++++---------- src/util/BufferPool.hpp | 4 ++ src/voxels/ChunksStorage.cpp | 7 +-- 7 files changed, 170 insertions(+), 86 deletions(-) create mode 100644 src/coders/compression.cpp create mode 100644 src/coders/compression.hpp diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp new file mode 100644 index 00000000..d32690a2 --- /dev/null +++ b/src/coders/compression.cpp @@ -0,0 +1,87 @@ +#include "compression.hpp" + +#include +#include + +#include "rle.hpp" +#include "gzip.hpp" +#include "util/BufferPool.hpp" + +using namespace compression; + +static util::BufferPool buffer_pools[] { + {255}, + {UINT16_MAX}, + {UINT16_MAX * 8}, +}; + +static std::shared_ptr get_buffer(size_t minSize) { + for (auto& pool : buffer_pools) { + if (minSize <= pool.getBufferSize()) { + return pool.get(); + } + } + return nullptr; +} + +std::unique_ptr compression::compress( + const ubyte* src, size_t srclen, size_t& len, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: { + // max extrle out size is srcLen * 2 + auto buffer = get_buffer(srclen * 2); + auto bytes = buffer.get(); + std::unique_ptr uptr; + if (bytes == nullptr) { + uptr = std::make_unique(srclen * 2); + bytes = uptr.get(); + } + len = extrle::encode(src, srclen, bytes); + if (uptr) { + return uptr; + } + auto data = std::make_unique(len); + std::memcpy(data.get(), bytes, len); + return data; + } + case Method::GZIP: { + auto buffer = gzip::compress(src, srclen); + auto data = std::make_unique(buffer.size()); + std::memcpy(data.get(), buffer.data(), buffer.size()); + len = buffer.size(); + return data; + } + default: + throw std::runtime_error("not implemented"); + } +} + +std::unique_ptr compression::decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: { + auto decompressed = std::make_unique(dstlen); + extrle::decode(src, srclen, decompressed.get()); + return decompressed; + } + case Method::GZIP: { + auto buffer = gzip::decompress(src, srclen); + if (buffer.size() != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(buffer.size())); + } + auto decompressed = std::make_unique(buffer.size()); + std::memcpy(decompressed.get(), buffer.data(), buffer.size()); + return decompressed; + } + default: + throw std::runtime_error("not implemented"); + } +} diff --git a/src/coders/compression.hpp b/src/coders/compression.hpp new file mode 100644 index 00000000..09f50b2a --- /dev/null +++ b/src/coders/compression.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "typedefs.hpp" + +namespace compression { + enum class Method { + NONE, EXTRLE8, GZIP + }; + + /// @brief Compress buffer + /// @param src source buffer + /// @param srclen length of the source buffer + /// @param len (out argument) length of result buffer + /// @param method compression method + /// @return compressed bytes array + /// @throws std::invalid_argument if compression method is NONE + std::unique_ptr compress( + const ubyte* src, size_t srclen, size_t& len, Method method + ); + + /// @brief Decompress buffer + /// @param src compressed buffer + /// @param srclen length of compressed buffer + /// @param dstlen max expected length of source buffer + /// @return decompressed bytes array + std::unique_ptr decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method); +} diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp index 2fc405de..2ec8782a 100644 --- a/src/files/RegionsLayer.cpp +++ b/src/files/RegionsLayer.cpp @@ -11,7 +11,7 @@ static fs::path get_region_filename(int x, int z) { } /// @brief Read missing chunks data (null pointers) from region file -static void fetchChunks(WorldRegion* region, int x, int z, regfile* file) { +static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) { auto* chunks = region->getChunks(); uint32_t* sizes = region->getSizes(); @@ -174,7 +174,7 @@ void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) { glm::ivec2 regcoord(x, z); if (auto regfile = getRegFile(regcoord, false)) { - fetchChunks(entry, x, z, regfile.get()); + fetch_chunks(entry, x, z, regfile.get()); std::lock_guard lock(regFilesMutex); regfile.reset(); diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 9074c69e..07400895 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -5,7 +5,6 @@ #include #include "coders/byte_utils.hpp" -#include "coders/rle.hpp" #include "data/dynamic.hpp" #include "items/Inventory.hpp" #include "maths/voxmaths.hpp" @@ -55,8 +54,14 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) { layers[i].layer = i; } - layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions"); - layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights"); + auto& voxels = layers[REGION_LAYER_VOXELS]; + voxels.folder = directory / fs::path("regions"); + voxels.compression = compression::Method::EXTRLE8; + + auto& lights = layers[REGION_LAYER_LIGHTS]; + lights.folder = directory / fs::path("lights"); + lights.compression = compression::Method::EXTRLE8; + layers[REGION_LAYER_INVENTORIES].folder = directory / fs::path("inventories"); layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities"); @@ -64,28 +69,6 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { WorldRegions::~WorldRegions() = default; -std::unique_ptr WorldRegions::compress( - const ubyte* src, size_t srclen, size_t& len -) { - auto buffer = bufferPool.get(); - auto bytes = buffer.get(); - - len = extrle::encode(src, srclen, bytes); - auto data = std::make_unique(len); - for (size_t i = 0; i < len; i++) { - data[i] = bytes[i]; - } - return data; -} - -std::unique_ptr WorldRegions::decompress( - const ubyte* src, size_t srclen, size_t dstlen -) { - auto decompressed = std::make_unique(dstlen); - extrle::decode(src, srclen, decompressed.get()); - return decompressed; -} - void RegionsLayer::writeAll() { for (auto& it : regions) { WorldRegion* region = it.second.get(); @@ -100,21 +83,19 @@ void RegionsLayer::writeAll() { void WorldRegions::put( int x, int z, - int layer, + int layerid, std::unique_ptr data, - size_t size, - bool rle + size_t size ) { - if (rle) { - size_t compressedSize; - auto compressed = compress(data.get(), size, compressedSize); - put(x, z, layer, std::move(compressed), compressedSize, false); - return; + auto& layer = layers[layerid]; + if (layer.compression != compression::Method::NONE) { + data = compression::compress( + data.get(), size, size, layer.compression); } int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - WorldRegion* region = layers[layer].getOrCreateRegion(regionX, regionZ); + WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); region->put(localX, localZ, std::move(data), size); } @@ -139,7 +120,6 @@ static std::unique_ptr write_inventories( return data; } -/// @brief Store chunk data (voxels and lights) in region (existing or new) void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { assert(chunk != nullptr); if (!chunk->flags.lighted) { @@ -157,8 +137,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_VOXELS, chunk->encode(), - CHUNK_DATA_LEN, - true); + CHUNK_DATA_LEN); // Writing lights cache if (doWriteLights && chunk->flags.lighted) { @@ -166,8 +145,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_LIGHTS, chunk->lightmap.encode(), - LIGHTMAP_DATA_LEN, - true); + LIGHTMAP_DATA_LEN); } // Writing block inventories if (!chunk->inventories.empty()) { @@ -177,8 +155,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_INVENTORIES, std::move(data), - datasize, - false); + datasize); } // Writing entities if (!entitiesData.empty()) { @@ -190,29 +167,30 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_ENTITIES, std::move(data), - entitiesData.size(), - false); + entitiesData.size()); } } std::unique_ptr WorldRegions::getVoxels(int x, int z) { uint32_t size; - auto* data = layers[REGION_LAYER_VOXELS].getData(x, z, size); + auto& layer = layers[REGION_LAYER_VOXELS]; + auto* data = layer.getData(x, z, size); if (data == nullptr) { return nullptr; } - return decompress(data, size, CHUNK_DATA_LEN); + return compression::decompress(data, size, CHUNK_DATA_LEN, layer.compression); } -/// @brief Get cached lights for chunk at x,z -/// @return lights data or nullptr std::unique_ptr WorldRegions::getLights(int x, int z) { uint32_t size; - auto* bytes = layers[REGION_LAYER_LIGHTS].getData(x, z, size); + auto& layer = layers[REGION_LAYER_LIGHTS]; + auto* bytes = layer.getData(x, z, size); if (bytes == nullptr) { return nullptr; } - auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN); + auto data = compression::decompress( + bytes, size, LIGHTMAP_DATA_LEN, layer.compression + ); return Lightmap::decode(data.get()); } @@ -254,10 +232,11 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { } void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - if (layers[REGION_LAYER_VOXELS].getRegion(x, z)) { + auto& layer = layers[REGION_LAYER_VOXELS]; + if (layer.getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } - auto regfile = layers[REGION_LAYER_VOXELS].getRegFile({x, z}); + auto regfile = layer.getRegFile({x, z}); if (regfile == nullptr) { throw std::runtime_error("could not open region file"); } @@ -271,14 +250,15 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { if (data == nullptr) { continue; } - data = decompress(data.get(), length, CHUNK_DATA_LEN); + data = compression::decompress( + data.get(), length, CHUNK_DATA_LEN, layer.compression + ); if (func(data.get())) { put(gx, gz, REGION_LAYER_VOXELS, std::move(data), - CHUNK_DATA_LEN, - true); + CHUNK_DATA_LEN); } } } diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index e467b798..df77fdb2 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -8,11 +8,12 @@ #include #include -#include "data/dynamic_fwd.hpp" #include "typedefs.hpp" +#include "data/dynamic_fwd.hpp" #include "util/BufferPool.hpp" #include "voxels/Chunk.hpp" #include "maths/voxmaths.hpp" +#include "coders/compression.hpp" #include "files.hpp" #define GLM_ENABLE_EXPERIMENTAL @@ -126,6 +127,8 @@ struct RegionsLayer { /// @brief Regions layer folder fs::path folder; + compression::Method compression = compression::Method::NONE; + /// @brief In-memory regions data regionsmap regions; @@ -139,7 +142,7 @@ struct RegionsLayer { std::mutex regFilesMutex; std::condition_variable regFilesCv; - regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); + [[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord); regfile_ptr createRegFile(glm::ivec2 coord); void closeRegFile(glm::ivec2 coord); @@ -178,26 +181,6 @@ class WorldRegions { fs::path directory; RegionsLayer layers[4] {}; - util::BufferPool bufferPool { - std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2}; - - /// @brief Compress buffer with extrle - /// @param src source buffer - /// @param srclen length of the source buffer - /// @param len (out argument) length of result buffer - /// @return compressed bytes array - std::unique_ptr compress( - const ubyte* src, size_t srclen, size_t& len - ); - - /// @brief Decompress buffer with extrle - /// @param src compressed buffer - /// @param srclen length of compressed buffer - /// @param dstlen max expected length of source buffer - /// @return decompressed bytes array - std::unique_ptr decompress( - const ubyte* src, size_t srclen, size_t dstlen - ); public: bool generatorTestMode = false; bool doWriteLights = true; @@ -215,19 +198,22 @@ public: /// @param layer regions layer /// @param data target data /// @param size data size - /// @param rle compress with ext-RLE void put( int x, int z, int layer, std::unique_ptr data, - size_t size, - bool rle + size_t size ); std::unique_ptr getVoxels(int x, int z); + + /// @brief Get cached lights for chunk at x,z + /// @return lights data or nullptr std::unique_ptr getLights(int x, int z); + chunk_inventories_map fetchInventories(int x, int z); + dynamic::Map_sptr fetchEntities(int x, int z); void processRegionVoxels(int x, int z, const regionproc& func); diff --git a/src/util/BufferPool.hpp b/src/util/BufferPool.hpp index 0d775b90..aeeebb8c 100644 --- a/src/util/BufferPool.hpp +++ b/src/util/BufferPool.hpp @@ -35,5 +35,9 @@ namespace util { freeBuffers.push(ptr); }); } + + size_t getBufferSize() const { + return bufferSize; + } }; } diff --git a/src/voxels/ChunksStorage.cpp b/src/voxels/ChunksStorage.cpp index 23665b61..b430dbe4 100644 --- a/src/voxels/ChunksStorage.cpp +++ b/src/voxels/ChunksStorage.cpp @@ -59,8 +59,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { auto chunk = std::make_shared(x, z); store(chunk); - auto data = regions.getVoxels(chunk->x, chunk->z); - if (data) { + if (auto data = regions.getVoxels(chunk->x, chunk->z)) { chunk->decode(data.get()); auto invs = regions.fetchInventories(chunk->x, chunk->z); @@ -77,9 +76,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { } verifyLoadedChunk(level->content->getIndices(), chunk.get()); } - - auto lights = regions.getLights(chunk->x, chunk->z); - if (lights) { + if (auto lights = regions.getLights(chunk->x, chunk->z)) { chunk->lightmap.set(lights.get()); chunk->flags.loadedLights = true; } From 10e10955dcdcce79691be6225d325eabbcab2788 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 2 Sep 2024 08:01:43 +0300 Subject: [PATCH 21/61] add missing include --- src/coders/compression.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp index d32690a2..343328cf 100644 --- a/src/coders/compression.cpp +++ b/src/coders/compression.cpp @@ -1,5 +1,6 @@ #include "compression.hpp" +#include #include #include From 3dda512468af4b9c01c0947194d008753680654d Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 2 Sep 2024 08:32:12 +0300 Subject: [PATCH 22/61] replace regions layer index with enum --- src/constants.hpp | 7 +++++++ src/files/WorldRegions.cpp | 6 +++--- src/files/WorldRegions.hpp | 21 ++++++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/constants.hpp b/src/constants.hpp index 15b3e732..b1ae11d4 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -16,6 +16,12 @@ inline constexpr bool ENGINE_DEBUG_BUILD = true; inline const std::string ENGINE_VERSION_STRING = "0.23"; +/// @brief world regions format version +inline constexpr uint REGION_FORMAT_VERSION = 2; + +/// @brief max simultaneously open world region files +inline constexpr uint MAX_OPEN_REGION_FILES = 32; + inline constexpr blockid_t BLOCK_AIR = 0; inline constexpr itemid_t ITEM_EMPTY = 0; inline constexpr entityid_t ENTITY_NONE = 0; @@ -40,6 +46,7 @@ inline constexpr itemid_t ITEM_VOID = std::numeric_limits::max(); /// @brief max number of block definitions possible inline constexpr blockid_t MAX_BLOCKS = BLOCK_VOID; +/// @brief calculates a 1D array index from 3D array indices inline constexpr uint vox_index(uint x, uint y, uint z, uint w=CHUNK_W, uint d=CHUNK_D) { return (y * d + z) * w + x; } diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 07400895..cd0955c1 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -51,8 +51,8 @@ uint WorldRegion::getChunkDataSize(uint x, uint z) { } WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { - for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) { - layers[i].layer = i; + for (size_t i = 0; i < REGION_LAYERS_COUNT; i++) { + layers[i].layer = static_cast(i); } auto& voxels = layers[REGION_LAYER_VOXELS]; voxels.folder = directory / fs::path("regions"); @@ -83,7 +83,7 @@ void RegionsLayer::writeAll() { void WorldRegions::put( int x, int z, - int layerid, + RegionLayerIndex layerid, std::unique_ptr data, size_t size ) { diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index df77fdb2..eb2e803a 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -8,6 +8,7 @@ #include #include +#include "constants.hpp" #include "typedefs.hpp" #include "data/dynamic_fwd.hpp" #include "util/BufferPool.hpp" @@ -23,16 +24,18 @@ namespace fs = std::filesystem; inline constexpr uint REGION_HEADER_SIZE = 10; -inline constexpr uint REGION_LAYER_VOXELS = 0; -inline constexpr uint REGION_LAYER_LIGHTS = 1; -inline constexpr uint REGION_LAYER_INVENTORIES = 2; -inline constexpr uint REGION_LAYER_ENTITIES = 3; +enum RegionLayerIndex : uint { + REGION_LAYER_VOXELS = 0, + REGION_LAYER_LIGHTS, + REGION_LAYER_INVENTORIES, + REGION_LAYER_ENTITIES, + + REGION_LAYERS_COUNT +}; inline constexpr uint REGION_SIZE_BIT = 5; inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); -inline constexpr uint REGION_FORMAT_VERSION = 2; -inline constexpr uint MAX_OPEN_REGION_FILES = 32; class illegal_region_format : public std::runtime_error { public: @@ -122,7 +125,7 @@ inline void calc_reg_coords( struct RegionsLayer { /// @brief Layer index - int layer; + RegionLayerIndex layer; /// @brief Regions layer folder fs::path folder; @@ -180,7 +183,7 @@ class WorldRegions { /// @brief World directory fs::path directory; - RegionsLayer layers[4] {}; + RegionsLayer layers[REGION_LAYERS_COUNT] {}; public: bool generatorTestMode = false; bool doWriteLights = true; @@ -201,7 +204,7 @@ public: void put( int x, int z, - int layer, + RegionLayerIndex layer, std::unique_ptr data, size_t size ); From c1ef4dbe9f998dd58176846b4e117af651ae989d Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 2 Sep 2024 09:31:53 +0300 Subject: [PATCH 23/61] add content issues (WIP) --- src/content/Content.hpp | 16 +++++++------- src/content/ContentBuilder.hpp | 14 ++++++------- src/content/ContentLUT.cpp | 38 ++++++++++++++++++++++++++++------ src/content/ContentLUT.hpp | 37 ++++++++++++++++++++++++++------- src/content/content_fwd.hpp | 2 +- src/files/WorldFiles.cpp | 1 + 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/content/Content.hpp b/src/content/Content.hpp index 1931c262..e2275612 100644 --- a/src/content/Content.hpp +++ b/src/content/Content.hpp @@ -24,15 +24,15 @@ namespace rigging { class SkeletonConfig; } -constexpr const char* contenttype_name(contenttype type) { +constexpr const char* contenttype_name(ContentType type) { switch (type) { - case contenttype::none: + case ContentType::NONE: return "none"; - case contenttype::block: + case ContentType::BLOCK: return "block"; - case contenttype::item: + case ContentType::ITEM: return "item"; - case contenttype::entity: + case ContentType::ENTITY: return "entity"; default: return "unknown"; @@ -40,13 +40,13 @@ constexpr const char* contenttype_name(contenttype type) { } class namereuse_error : public std::runtime_error { - contenttype type; + ContentType type; public: - namereuse_error(const std::string& msg, contenttype type) + namereuse_error(const std::string& msg, ContentType type) : std::runtime_error(msg), type(type) { } - inline contenttype getType() const { + inline ContentType getType() const { return type; } }; diff --git a/src/content/ContentBuilder.hpp b/src/content/ContentBuilder.hpp index d3e8f536..7961d1bf 100644 --- a/src/content/ContentBuilder.hpp +++ b/src/content/ContentBuilder.hpp @@ -12,8 +12,8 @@ template class ContentUnitBuilder { - std::unordered_map& allNames; - contenttype type; + std::unordered_map& allNames; + ContentType type; void checkIdentifier(const std::string& id) { const auto& found = allNames.find(id); @@ -28,7 +28,7 @@ public: std::vector names; ContentUnitBuilder( - std::unordered_map& allNames, contenttype type + std::unordered_map& allNames, ContentType type ) : allNames(allNames), type(type) { } @@ -62,11 +62,11 @@ class ContentBuilder { UptrsMap blockMaterials; UptrsMap skeletons; UptrsMap packs; - std::unordered_map allNames; + std::unordered_map allNames; public: - ContentUnitBuilder blocks {allNames, contenttype::block}; - ContentUnitBuilder items {allNames, contenttype::item}; - ContentUnitBuilder entities {allNames, contenttype::entity}; + ContentUnitBuilder blocks {allNames, ContentType::BLOCK}; + ContentUnitBuilder items {allNames, ContentType::ITEM}; + ContentUnitBuilder entities {allNames, ContentType::ENTITY}; ResourceIndicesSet resourceIndices {}; ~ContentBuilder(); diff --git a/src/content/ContentLUT.cpp b/src/content/ContentLUT.cpp index 8e670152..80c0699c 100644 --- a/src/content/ContentLUT.cpp +++ b/src/content/ContentLUT.cpp @@ -12,11 +12,13 @@ #include "Content.hpp" ContentLUT::ContentLUT( - const ContentIndices* indices, size_t blocksCount, size_t itemsCount + const ContentIndices* indices, + size_t blocksCount, + size_t itemsCount ) - : blocks(blocksCount, indices->blocks, BLOCK_VOID, contenttype::block), - items(itemsCount, indices->items, ITEM_VOID, contenttype::item) { -} + : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK), + items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM) +{} template static constexpr size_t get_entries_count( @@ -47,6 +49,7 @@ std::shared_ptr ContentLUT::create( lut->blocks.setup(blocklist.get(), content->blocks); lut->items.setup(itemlist.get(), content->items); + lut->buildIssues(); if (lut->hasContentReorder() || lut->hasMissingContent()) { return lut; @@ -55,8 +58,31 @@ std::shared_ptr ContentLUT::create( } } -std::vector ContentLUT::getMissingContent() const { - std::vector entries; +template +static void build_issues( + std::vector& issues, + const ContentUnitLUT& lut +) { + auto type = lut.getContentType(); + if (lut.hasContentReorder()) { + issues.push_back(ContentIssue {ContentIssueType::REORDER, type}); + } + if (lut.hasMissingContent()) { + issues.push_back(ContentIssue {ContentIssueType::MISSING, type}); + } +} + +void ContentLUT::buildIssues() { + build_issues(issues, blocks); + build_issues(issues, items); +} + +const std::vector& ContentLUT::getIssues() const { + return issues; +} + +std::vector ContentLUT::getMissingContent() const { + std::vector entries; blocks.getMissingContent(entries); items.getMissingContent(entries); return entries; diff --git a/src/content/ContentLUT.hpp b/src/content/ContentLUT.hpp index f0b92c07..20259114 100644 --- a/src/content/ContentLUT.hpp +++ b/src/content/ContentLUT.hpp @@ -12,8 +12,18 @@ namespace fs = std::filesystem; -struct contententry { - contenttype type; +enum class ContentIssueType { + REORDER, + MISSING, +}; + +struct ContentIssue { + ContentIssueType issueType; + ContentType contentType; +}; + +struct ContentEntry { + ContentType type; std::string name; }; @@ -26,13 +36,13 @@ class ContentUnitLUT { bool missingContent = false; bool reorderContent = false; T missingValue; - contenttype type; + ContentType type; public: ContentUnitLUT( size_t count, const ContentUnitIndices& unitIndices, T missingValue, - contenttype type + ContentType type ) : missingValue(missingValue), type(type) { for (size_t i = 0; i < count; i++) { @@ -57,11 +67,11 @@ public: } } } - void getMissingContent(std::vector& entries) const { + void getMissingContent(std::vector& entries) const { for (size_t i = 0; i < count(); i++) { if (indices[i] == missingValue) { auto& name = names[i]; - entries.push_back(contententry {type, name}); + entries.push_back(ContentEntry {type, name}); } } } @@ -80,6 +90,9 @@ public: reorderContent = true; } } + inline ContentType getContentType() const { + return type; + } inline size_t count() const { return indices.size(); } @@ -99,7 +112,13 @@ public: ContentUnitLUT blocks; ContentUnitLUT items; - ContentLUT(const ContentIndices* indices, size_t blocks, size_t items); + std::vector issues; + + ContentLUT( + const ContentIndices* indices, + size_t blocks, + size_t items + ); static std::shared_ptr create( const std::shared_ptr& worldFiles, @@ -113,6 +132,8 @@ public: inline bool hasMissingContent() const { return blocks.hasMissingContent() || items.hasMissingContent(); } + void buildIssues(); - std::vector getMissingContent() const; + const std::vector& getIssues() const; + std::vector getMissingContent() const; }; diff --git a/src/content/content_fwd.hpp b/src/content/content_fwd.hpp index 719fbfca..b1618593 100644 --- a/src/content/content_fwd.hpp +++ b/src/content/content_fwd.hpp @@ -5,7 +5,7 @@ class Content; class ContentPackRuntime; -enum class contenttype { none, block, item, entity }; +enum class ContentType { NONE, BLOCK, ITEM, ENTITY }; enum class ResourceType : size_t { CAMERA, LAST = CAMERA }; diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 33ebaf37..01b9cb77 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -109,6 +109,7 @@ static void write_indices( void WorldFiles::writeIndices(const ContentIndices* indices) { dynamic::Map root; + root.put("region-version", REGION_FORMAT_VERSION); write_indices(indices->blocks, root.putList("blocks")); write_indices(indices->items, root.putList("items")); write_indices(indices->entities, root.putList("entities")); From 3f826a88d31dce8831939b577ad9e7b73526ef28 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 2 Sep 2024 09:40:00 +0300 Subject: [PATCH 24/61] rename ContentLUT to ContentReport --- .../{ContentLUT.cpp => ContentReport.cpp} | 33 +++++++++---------- .../{ContentLUT.hpp => ContentReport.hpp} | 9 +++-- src/files/WorldConverter.cpp | 16 ++++----- src/files/WorldConverter.hpp | 8 ++--- src/items/Inventory.cpp | 6 ++-- src/items/Inventory.hpp | 4 +-- src/logic/EngineController.cpp | 28 ++++++++-------- src/objects/Player.cpp | 8 ++--- src/objects/Player.hpp | 4 +-- src/voxels/Chunk.cpp | 6 ++-- src/voxels/Chunk.hpp | 4 +-- src/world/World.cpp | 6 ++-- src/world/World.hpp | 4 +-- 13 files changed, 67 insertions(+), 69 deletions(-) rename src/content/{ContentLUT.cpp => ContentReport.cpp} (67%) rename src/content/{ContentLUT.hpp => ContentReport.hpp} (91%) diff --git a/src/content/ContentLUT.cpp b/src/content/ContentReport.cpp similarity index 67% rename from src/content/ContentLUT.cpp rename to src/content/ContentReport.cpp index 80c0699c..54b4098d 100644 --- a/src/content/ContentLUT.cpp +++ b/src/content/ContentReport.cpp @@ -1,4 +1,4 @@ -#include "ContentLUT.hpp" +#include "ContentReport.hpp" #include @@ -11,7 +11,7 @@ #include "files/WorldFiles.hpp" #include "Content.hpp" -ContentLUT::ContentLUT( +ContentReport::ContentReport( const ContentIndices* indices, size_t blocksCount, size_t itemsCount @@ -27,7 +27,7 @@ static constexpr size_t get_entries_count( return list ? std::max(list->size(), indices.count()) : indices.count(); } -std::shared_ptr ContentLUT::create( +std::shared_ptr ContentReport::create( const std::shared_ptr& worldFiles, const fs::path& filename, const Content* content @@ -45,14 +45,13 @@ std::shared_ptr ContentLUT::create( size_t blocks_c = get_entries_count(indices->blocks, blocklist); size_t items_c = get_entries_count(indices->items, itemlist); - auto lut = std::make_shared(indices, blocks_c, items_c); + auto report = std::make_shared(indices, blocks_c, items_c); + report->blocks.setup(blocklist.get(), content->blocks); + report->items.setup(itemlist.get(), content->items); + report->buildIssues(); - lut->blocks.setup(blocklist.get(), content->blocks); - lut->items.setup(itemlist.get(), content->items); - lut->buildIssues(); - - if (lut->hasContentReorder() || lut->hasMissingContent()) { - return lut; + if (report->hasContentReorder() || report->hasMissingContent()) { + return report; } else { return nullptr; } @@ -61,27 +60,27 @@ std::shared_ptr ContentLUT::create( template static void build_issues( std::vector& issues, - const ContentUnitLUT& lut + const ContentUnitLUT& report ) { - auto type = lut.getContentType(); - if (lut.hasContentReorder()) { + auto type = report.getContentType(); + if (report.hasContentReorder()) { issues.push_back(ContentIssue {ContentIssueType::REORDER, type}); } - if (lut.hasMissingContent()) { + if (report.hasMissingContent()) { issues.push_back(ContentIssue {ContentIssueType::MISSING, type}); } } -void ContentLUT::buildIssues() { +void ContentReport::buildIssues() { build_issues(issues, blocks); build_issues(issues, items); } -const std::vector& ContentLUT::getIssues() const { +const std::vector& ContentReport::getIssues() const { return issues; } -std::vector ContentLUT::getMissingContent() const { +std::vector ContentReport::getMissingContent() const { std::vector entries; blocks.getMissingContent(entries); items.getMissingContent(entries); diff --git a/src/content/ContentLUT.hpp b/src/content/ContentReport.hpp similarity index 91% rename from src/content/ContentLUT.hpp rename to src/content/ContentReport.hpp index 20259114..d4c1a056 100644 --- a/src/content/ContentLUT.hpp +++ b/src/content/ContentReport.hpp @@ -104,23 +104,22 @@ public: } }; -/// @brief Content indices lookup table or report -/// used to convert world with different indices +/// @brief Content incapatibility report used to convert world. /// Building with indices.json -class ContentLUT { +class ContentReport { public: ContentUnitLUT blocks; ContentUnitLUT items; std::vector issues; - ContentLUT( + ContentReport( const ContentIndices* indices, size_t blocks, size_t items ); - static std::shared_ptr create( + static std::shared_ptr create( const std::shared_ptr& worldFiles, const fs::path& filename, const Content* content diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index 8e23e432..c94708dd 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -5,7 +5,7 @@ #include #include -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" #include "data/dynamic.hpp" #include "debug/Logger.hpp" #include "files/files.hpp" @@ -34,10 +34,10 @@ public: WorldConverter::WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr lut + std::shared_ptr report ) : wfile(worldFiles), - lut(std::move(lut)), + report(std::move(report)), content(content) { fs::path regionsFolder = wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS); @@ -58,11 +58,11 @@ WorldConverter::~WorldConverter() { std::shared_ptr WorldConverter::startTask( const std::shared_ptr& worldFiles, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const runnable& onDone, bool multithreading ) { - auto converter = std::make_shared(worldFiles, content, lut); + auto converter = std::make_shared(worldFiles, content, report); if (!multithreading) { converter->setOnComplete([=]() { converter->write(); @@ -98,8 +98,8 @@ void WorldConverter::convertRegion(const fs::path& file) const { } logger.info() << "converting region " << name; wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) { - if (lut) { - Chunk::convert(data, lut.get()); + if (report) { + Chunk::convert(data, report.get()); } return true; }); @@ -108,7 +108,7 @@ void WorldConverter::convertRegion(const fs::path& file) const { void WorldConverter::convertPlayer(const fs::path& file) const { logger.info() << "converting player " << file.u8string(); auto map = files::read_json(file); - Player::convert(map.get(), lut.get()); + Player::convert(map.get(), report.get()); files::write_json(file, map.get()); } diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index c16151e8..3b14942a 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -11,7 +11,7 @@ namespace fs = std::filesystem; class Content; -class ContentLUT; +class ContentReport; class WorldFiles; enum class convert_task_type { region, player }; @@ -23,7 +23,7 @@ struct convert_task { class WorldConverter : public Task { std::shared_ptr wfile; - std::shared_ptr const lut; + std::shared_ptr const report; const Content* const content; std::queue tasks; runnable onComplete; @@ -35,7 +35,7 @@ public: WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr lut + std::shared_ptr report ); ~WorldConverter(); @@ -54,7 +54,7 @@ public: static std::shared_ptr startTask( const std::shared_ptr& worldFiles, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const runnable& onDone, bool multithreading ); diff --git a/src/items/Inventory.cpp b/src/items/Inventory.cpp index 34f20bba..c2bc16ed 100644 --- a/src/items/Inventory.cpp +++ b/src/items/Inventory.cpp @@ -1,6 +1,6 @@ #include "Inventory.hpp" -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" #include "data/dynamic.hpp" Inventory::Inventory(int64_t id, size_t size) : id(id), slots(size) { @@ -81,12 +81,12 @@ std::unique_ptr Inventory::serialize() const { return map; } -void Inventory::convert(dynamic::Map* data, const ContentLUT* lut) { +void Inventory::convert(dynamic::Map* data, const ContentReport* report) { auto slotsarr = data->list("slots"); for (size_t i = 0; i < slotsarr->size(); i++) { auto item = slotsarr->map(i); itemid_t id = item->get("id", ITEM_EMPTY); - itemid_t replacement = lut->items.getId(id); + itemid_t replacement = report->items.getId(id); item->put("id", replacement); if (replacement == 0 && item->has("count")) { item->remove("count"); diff --git a/src/items/Inventory.hpp b/src/items/Inventory.hpp index a557d2c0..260ec08c 100644 --- a/src/items/Inventory.hpp +++ b/src/items/Inventory.hpp @@ -11,7 +11,7 @@ namespace dynamic { class Map; } -class ContentLUT; +class ContentReport; class ContentIndices; class Inventory : public Serializable { @@ -42,7 +42,7 @@ public: /* serializing inventory */ std::unique_ptr serialize() const override; - static void convert(dynamic::Map* data, const ContentLUT* lut); + static void convert(dynamic::Map* data, const ContentReport* report); inline void setId(int64_t id) { this->id = id; diff --git a/src/logic/EngineController.cpp b/src/logic/EngineController.cpp index 7e0d59c3..91e77220 100644 --- a/src/logic/EngineController.cpp +++ b/src/logic/EngineController.cpp @@ -4,10 +4,10 @@ #include #include -#include "coders/commons.hpp" -#include "content/ContentLUT.hpp" -#include "debug/Logger.hpp" #include "engine.hpp" +#include "coders/commons.hpp" +#include "debug/Logger.hpp" +#include "content/ContentReport.hpp" #include "files/WorldConverter.hpp" #include "files/WorldFiles.hpp" #include "frontend/locale.hpp" @@ -46,13 +46,13 @@ std::shared_ptr create_converter( Engine* engine, const std::shared_ptr& worldFiles, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const runnable& postRunnable ) { return WorldConverter::startTask( worldFiles, content, - lut, + report, [=]() { auto menu = engine->getGUI()->getMenu(); menu->reset(); @@ -66,7 +66,7 @@ std::shared_ptr create_converter( void show_convert_request( Engine* engine, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const std::shared_ptr& worldFiles, const runnable& postRunnable ) { @@ -75,7 +75,7 @@ void show_convert_request( langs::get(L"world.convert-request"), [=]() { auto converter = - create_converter(engine, worldFiles, content, lut, postRunnable); + create_converter(engine, worldFiles, content, report, postRunnable); menus::show_process_panel( engine, converter, L"Converting world..." ); @@ -86,12 +86,12 @@ void show_convert_request( } static void show_content_missing( - Engine* engine, const std::shared_ptr& lut + Engine* engine, const std::shared_ptr& report ) { using namespace dynamic; auto root = create_map(); auto& contentEntries = root->putList("content"); - for (auto& entry : lut->getMissingContent()) { + for (auto& entry : report->getMissingContent()) { std::string contentName = contenttype_name(entry.type); auto& contentEntry = contentEntries.putMap(); contentEntry.put("type", contentName); @@ -135,10 +135,10 @@ void EngineController::openWorld(const std::string& name, bool confirmConvert) { auto* content = engine->getContent(); auto worldFiles = std::make_shared( folder, engine->getSettings().debug); - if (auto lut = World::checkIndices(worldFiles, content)) { - if (lut->hasMissingContent()) { + if (auto report = World::checkIndices(worldFiles, content)) { + if (report->hasMissingContent()) { engine->setScreen(std::make_shared(engine)); - show_content_missing(engine, lut); + show_content_missing(engine, report); } else { if (confirmConvert) { menus::show_process_panel( @@ -147,13 +147,13 @@ void EngineController::openWorld(const std::string& name, bool confirmConvert) { engine, worldFiles, content, - lut, + report, [=]() { openWorld(name, false); } ), L"Converting world..." ); } else { - show_convert_request(engine, content, lut, std::move(worldFiles), [=]() { + show_convert_request(engine, content, report, std::move(worldFiles), [=]() { openWorld(name, false); }); } diff --git a/src/objects/Player.cpp b/src/objects/Player.cpp index 74fad257..3c145473 100644 --- a/src/objects/Player.cpp +++ b/src/objects/Player.cpp @@ -4,7 +4,7 @@ #include #include -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" #include "items/Inventory.hpp" #include "Entities.hpp" #include "rigging.hpp" @@ -329,19 +329,19 @@ void Player::deserialize(dynamic::Map* src) { } } -void Player::convert(dynamic::Map* data, const ContentLUT* lut) { +void Player::convert(dynamic::Map* data, const ContentReport* report) { auto players = data->list("players"); if (players) { for (uint i = 0; i < players->size(); i++) { auto playerData = players->map(i); if (auto inventory = playerData->map("inventory")) { - Inventory::convert(inventory.get(), lut); + Inventory::convert(inventory.get(), report); } } } else { if (auto inventory = data->map("inventory")) { - Inventory::convert(inventory.get(), lut); + Inventory::convert(inventory.get(), report); } } } diff --git a/src/objects/Player.hpp b/src/objects/Player.hpp index 94ce3096..49c18fac 100644 --- a/src/objects/Player.hpp +++ b/src/objects/Player.hpp @@ -12,7 +12,7 @@ class Camera; class Inventory; -class ContentLUT; +class ContentReport; class Level; struct Hitbox; struct EngineSettings; @@ -106,7 +106,7 @@ public: std::unique_ptr serialize() const override; void deserialize(dynamic::Map* src) override; - static void convert(dynamic::Map* data, const ContentLUT* lut); + static void convert(dynamic::Map* data, const ContentReport* report); inline int getId() const { return objectUID; diff --git a/src/voxels/Chunk.cpp b/src/voxels/Chunk.cpp index 78932659..c77532da 100644 --- a/src/voxels/Chunk.cpp +++ b/src/voxels/Chunk.cpp @@ -2,7 +2,7 @@ #include -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" #include "items/Inventory.hpp" #include "lighting/Lightmap.hpp" #include "voxel.hpp" @@ -123,13 +123,13 @@ bool Chunk::decode(const ubyte* data) { return true; } -void Chunk::convert(ubyte* data, const ContentLUT* lut) { +void Chunk::convert(ubyte* data, const ContentReport* report) { for (uint i = 0; i < CHUNK_VOL; i++) { // see encode method to understand what the hell is going on here blockid_t id = ((static_cast(data[i]) << 8) | static_cast(data[CHUNK_VOL + i])); - blockid_t replacement = lut->blocks.getId(id); + blockid_t replacement = report->blocks.getId(id); data[i] = replacement >> 8; data[CHUNK_VOL + i] = replacement & 0xFF; } diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index 14ead964..d9a25186 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -12,7 +12,7 @@ inline constexpr int CHUNK_DATA_LEN = CHUNK_VOL * 4; class Lightmap; -class ContentLUT; +class ContentReport; class Inventory; namespace dynamic { @@ -71,5 +71,5 @@ public: /// @return true if all is fine bool decode(const ubyte* data); - static void convert(ubyte* data, const ContentLUT* lut); + static void convert(ubyte* data, const ContentReport* report); }; diff --git a/src/world/World.cpp b/src/world/World.cpp index f8358960..c3e9e07c 100644 --- a/src/world/World.cpp +++ b/src/world/World.cpp @@ -5,7 +5,7 @@ #include #include "content/Content.hpp" -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" #include "debug/Logger.hpp" #include "files/WorldFiles.hpp" #include "items/Inventories.hpp" @@ -154,12 +154,12 @@ std::unique_ptr World::load( return level; } -std::shared_ptr World::checkIndices( +std::shared_ptr World::checkIndices( const std::shared_ptr& worldFiles, const Content* content ) { fs::path indicesFile = worldFiles->getIndicesFile(); if (fs::is_regular_file(indicesFile)) { - return ContentLUT::create(worldFiles, indicesFile, content); + return ContentReport::create(worldFiles, indicesFile, content); } return nullptr; } diff --git a/src/world/World.hpp b/src/world/World.hpp index f5443b2a..b166ee1f 100644 --- a/src/world/World.hpp +++ b/src/world/World.hpp @@ -14,7 +14,7 @@ class Content; class WorldFiles; class Level; -class ContentLUT; +class ContentReport; struct EngineSettings; namespace fs = std::filesystem; @@ -85,7 +85,7 @@ public: /// @param directory world directory /// @param content current Content instance /// @return ContentLUT if world convert required else nullptr - static std::shared_ptr checkIndices( + static std::shared_ptr checkIndices( const std::shared_ptr& worldFiles, const Content* content ); From 728795f0f33625ca2ee2e3e406b907aef7b58970 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 2 Sep 2024 23:24:59 +0300 Subject: [PATCH 25/61] update WorldConverter (WIP) --- src/content/ContentReport.cpp | 11 ++- src/content/ContentReport.hpp | 18 +++- src/files/WorldConverter.cpp | 163 ++++++++++++++++++++++++-------- src/files/WorldConverter.hpp | 43 +++++++-- src/files/WorldFiles.cpp | 10 +- src/files/WorldRegions.cpp | 84 ++++++++++------ src/files/WorldRegions.hpp | 41 +++++--- src/files/world_regions_fwd.hpp | 12 +++ src/items/Inventory.cpp | 9 ++ src/items/Inventory.hpp | 1 + src/items/ItemStack.cpp | 3 + src/logic/EngineController.cpp | 1 + 12 files changed, 299 insertions(+), 97 deletions(-) create mode 100644 src/files/world_regions_fwd.hpp diff --git a/src/content/ContentReport.cpp b/src/content/ContentReport.cpp index 54b4098d..1e524309 100644 --- a/src/content/ContentReport.cpp +++ b/src/content/ContentReport.cpp @@ -14,10 +14,12 @@ ContentReport::ContentReport( const ContentIndices* indices, size_t blocksCount, - size_t itemsCount + size_t itemsCount, + uint regionsVersion ) : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK), - items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM) + items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM), + regionsVersion(regionsVersion) {} template @@ -38,6 +40,8 @@ std::shared_ptr ContentReport::create( } auto root = files::read_json(filename); + // TODO: remove default value 2 in 0.24 + uint regionsVersion = root->get("region-version", 2U); auto blocklist = root->list("blocks"); auto itemlist = root->list("items"); @@ -45,7 +49,8 @@ std::shared_ptr ContentReport::create( size_t blocks_c = get_entries_count(indices->blocks, blocklist); size_t items_c = get_entries_count(indices->items, itemlist); - auto report = std::make_shared(indices, blocks_c, items_c); + auto report = std::make_shared( + indices, blocks_c, items_c, regionsVersion); report->blocks.setup(blocklist.get(), content->blocks); report->items.setup(itemlist.get(), content->items); report->buildIssues(); diff --git a/src/content/ContentReport.hpp b/src/content/ContentReport.hpp index d4c1a056..416be01e 100644 --- a/src/content/ContentReport.hpp +++ b/src/content/ContentReport.hpp @@ -9,17 +9,22 @@ #include "data/dynamic.hpp" #include "typedefs.hpp" #include "Content.hpp" +#include "files/world_regions_fwd.hpp" namespace fs = std::filesystem; enum class ContentIssueType { REORDER, MISSING, + REGION_FORMAT_UPDATE, }; struct ContentIssue { ContentIssueType issueType; - ContentType contentType; + union { + ContentType contentType; + RegionLayerIndex regionLayer; + }; }; struct ContentEntry { @@ -29,12 +34,16 @@ struct ContentEntry { class WorldFiles; +/// @brief Content unit lookup table +/// @tparam T index type +/// @tparam U unit class template class ContentUnitLUT { std::vector indices; std::vector names; bool missingContent = false; bool reorderContent = false; + /// @brief index that will be used to mark missing unit T missingValue; ContentType type; public: @@ -110,13 +119,15 @@ class ContentReport { public: ContentUnitLUT blocks; ContentUnitLUT items; + uint regionsVersion; std::vector issues; ContentReport( const ContentIndices* indices, size_t blocks, - size_t items + size_t items, + uint regionsVersion ); static std::shared_ptr create( @@ -131,6 +142,9 @@ public: inline bool hasMissingContent() const { return blocks.hasMissingContent() || items.hasMissingContent(); } + inline bool isUpgradeRequired() const { + return regionsVersion < REGION_FORMAT_VERSION; + } void buildIssues(); const std::vector& getIssues() const; diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index c94708dd..a6b977bb 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -12,43 +12,111 @@ #include "objects/Player.hpp" #include "util/ThreadPool.hpp" #include "voxels/Chunk.hpp" +#include "items/Inventory.hpp" #include "WorldFiles.hpp" namespace fs = std::filesystem; static debug::Logger logger("world-converter"); -class ConverterWorker : public util::Worker { +class ConverterWorker : public util::Worker { std::shared_ptr converter; public: ConverterWorker(std::shared_ptr converter) : converter(std::move(converter)) { } - int operator()(const std::shared_ptr& task) override { + int operator()(const std::shared_ptr& task) override { converter->convert(*task); return 0; } }; +void WorldConverter::addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType +) { + const auto& regions = wfile->getRegions(); + auto regionsFolder = regions.getRegionsFolder(layerid); + if (!fs::is_directory(regionsFolder)) { + return; + } + for (const auto& file : fs::directory_iterator(regionsFolder)) { + int x, z; + std::string name = file.path().stem().string(); + if (!WorldRegions::parseRegionFilename(name, x, z)) { + logger.error() << "could not parse region name " << name; + continue; + } + tasks.push(ConvertTask {taskType, file.path(), x, z}); + } +} + +void WorldConverter::createUpgradeTasks() { + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) { + continue; + } + if (issue.regionLayer == REGION_LAYER_VOXELS) { + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS); + } else { + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_SIMPLE); + } + } +} + +void WorldConverter::createConvertTasks() { + auto handleReorder = [=](ContentType contentType) { + switch (contentType) { + case ContentType::BLOCK: + addRegionsTasks( + REGION_LAYER_VOXELS, + ConvertTaskType::VOXELS + ); + break; + case ContentType::ITEM: + addRegionsTasks( + REGION_LAYER_INVENTORIES, + ConvertTaskType::INVENTORIES + ); + break; + default: + break; + } + }; + + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + switch (issue.issueType) { + case ContentIssueType::REGION_FORMAT_UPDATE: + break; + case ContentIssueType::MISSING: + throw std::runtime_error("issue can't be resolved"); + case ContentIssueType::REORDER: + handleReorder(issue.contentType); + break; + } + } + + tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()}); +} + WorldConverter::WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr report + std::shared_ptr reportPtr, + bool upgradeMode ) : wfile(worldFiles), - report(std::move(report)), - content(content) { - fs::path regionsFolder = - wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS); - if (!fs::is_directory(regionsFolder)) { - logger.error() << "nothing to convert"; - return; - } - tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()} - ); - for (const auto& file : fs::directory_iterator(regionsFolder)) { - tasks.push(convert_task {convert_task_type::region, file.path()}); + report(std::move(reportPtr)), + content(content), + upgradeMode(upgradeMode) +{ + if (upgradeMode) { + createUpgradeTasks(); + } else { + createConvertTasks(); } } @@ -60,9 +128,11 @@ std::shared_ptr WorldConverter::startTask( const Content* content, const std::shared_ptr& report, const runnable& onDone, + bool upgradeMode, bool multithreading ) { - auto converter = std::make_shared(worldFiles, content, report); + auto converter = std::make_shared( + worldFiles, content, report, upgradeMode); if (!multithreading) { converter->setOnComplete([=]() { converter->write(); @@ -70,15 +140,15 @@ std::shared_ptr WorldConverter::startTask( }); return converter; } - auto pool = std::make_shared>( + auto pool = std::make_shared>( "converter-pool", [=]() { return std::make_shared(converter); }, [=](int&) {} ); auto& converterTasks = converter->tasks; while (!converterTasks.empty()) { - const convert_task& task = converterTasks.front(); - auto ptr = std::make_shared(task); + const ConvertTask& task = converterTasks.front(); + auto ptr = std::make_shared(task); pool->enqueueJob(ptr); converterTasks.pop(); } @@ -89,19 +159,27 @@ std::shared_ptr WorldConverter::startTask( return pool; } -void WorldConverter::convertRegion(const fs::path& file) const { - int x, z; - std::string name = file.stem().string(); - if (!WorldRegions::parseRegionFilename(name, x, z)) { - logger.error() << "could not parse name " << name; - return; - } - logger.info() << "converting region " << name; - wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) { - if (report) { - Chunk::convert(data, report.get()); - } - return true; +void WorldConverter::upgradeSimple(const fs::path& file, int x, int z) const { + throw std::runtime_error("unsupported region format"); +} + +void WorldConverter::upgradeVoxels(const fs::path& file, int x, int z) const { + throw std::runtime_error("unsupported region format"); +} + +void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const { + logger.info() << "converting voxels region " << x << "_" << z; + wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, CHUNK_DATA_LEN, + [=](std::unique_ptr data, uint32_t*) { + Chunk::convert(data.get(), report.get()); + return data; + }); +} + +void WorldConverter::convertInventories(const fs::path& file, int x, int z) const { + logger.info() << "converting inventories region " << x << "_" << z; + wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) { + inventory->convert(report.get()); }); } @@ -112,14 +190,23 @@ void WorldConverter::convertPlayer(const fs::path& file) const { files::write_json(file, map.get()); } -void WorldConverter::convert(const convert_task& task) const { +void WorldConverter::convert(const ConvertTask& task) const { if (!fs::is_regular_file(task.file)) return; switch (task.type) { - case convert_task_type::region: - convertRegion(task.file); + case ConvertTaskType::UPGRADE_SIMPLE: + upgradeSimple(task.file, task.x, task.z); break; - case convert_task_type::player: + case ConvertTaskType::UPGRADE_VOXELS: + upgradeVoxels(task.file, task.x, task.z); + break; + case ConvertTaskType::VOXELS: + convertVoxels(task.file, task.x, task.z); + break; + case ConvertTaskType::INVENTORIES: + convertInventories(task.file, task.x, task.z); + break; + case ConvertTaskType::PLAYER: convertPlayer(task.file); break; } @@ -129,7 +216,7 @@ void WorldConverter::convertNext() { if (tasks.empty()) { throw std::runtime_error("no more regions to convert"); } - convert_task task = tasks.front(); + ConvertTask task = tasks.front(); tasks.pop(); tasksDone++; @@ -157,7 +244,7 @@ bool WorldConverter::isActive() const { void WorldConverter::write() { logger.info() << "writing world"; - wfile->write(nullptr, content); + wfile->write(nullptr, upgradeMode ? nullptr : content); } void WorldConverter::waitForEnd() { diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index 3b14942a..d8a3ce5f 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -6,6 +6,7 @@ #include "delegates.hpp" #include "interfaces/Task.hpp" +#include "files/world_regions_fwd.hpp" #include "typedefs.hpp" namespace fs = std::filesystem; @@ -14,32 +15,59 @@ class Content; class ContentReport; class WorldFiles; -enum class convert_task_type { region, player }; +enum class ConvertTaskType { + /// @brief rewrite voxels region indices + VOXELS, + /// @brief rewrite inventories region indices + INVENTORIES, + /// @brief rewrite player + PLAYER, + /// @brief refresh region file version + UPGRADE_SIMPLE, + /// @brief rewrite voxels region file to new format + UPGRADE_VOXELS, +}; -struct convert_task { - convert_task_type type; +struct ConvertTask { + ConvertTaskType type; fs::path file; + + /// @brief region coords + int x, z; }; class WorldConverter : public Task { std::shared_ptr wfile; std::shared_ptr const report; const Content* const content; - std::queue tasks; + std::queue tasks; runnable onComplete; uint tasksDone = 0; + bool upgradeMode; + void upgradeSimple(const fs::path& file, int x, int z) const; + void upgradeVoxels(const fs::path& file, int x, int z) const; void convertPlayer(const fs::path& file) const; - void convertRegion(const fs::path& file) const; + void convertVoxels(const fs::path& file, int x, int z) const; + void convertInventories(const fs::path& file, int x, int z) const; + + void addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType + ); + + void createUpgradeTasks(); + void createConvertTasks(); public: WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr report + std::shared_ptr report, + bool upgradeMode ); ~WorldConverter(); - void convert(const convert_task& task) const; + void convert(const ConvertTask& task) const; void convertNext(); void setOnComplete(runnable callback); void write(); @@ -56,6 +84,7 @@ public: const Content* content, const std::shared_ptr& report, const runnable& onDone, + bool upgradeMode, bool multithreading ); }; diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 01b9cb77..54e96759 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -74,7 +74,9 @@ fs::path WorldFiles::getPacksFile() const { return directory / fs::path("packs.list"); } -void WorldFiles::write(const World* world, const Content* content) { +void WorldFiles::write( + const World* world, const Content* content +) { if (world) { writeWorldInfo(world->getInfo()); if (!fs::exists(getPacksFile())) { @@ -84,8 +86,10 @@ void WorldFiles::write(const World* world, const Content* content) { if (generatorTestMode) { return; } - writeIndices(content->getIndices()); - regions.write(); + if (content) { + writeIndices(content->getIndices()); + } + regions.writeAll(); } void WorldFiles::writePacks(const std::vector& packs) { diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index cd0955c1..f3e1fd88 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -101,9 +101,8 @@ void WorldRegions::put( } static std::unique_ptr write_inventories( - Chunk* chunk, uint& datasize + const chunk_inventories_map& inventories, uint32_t& datasize ) { - auto& inventories = chunk->inventories; ByteBuilder builder; builder.putInt32(inventories.size()); for (auto& entry : inventories) { @@ -120,6 +119,22 @@ static std::unique_ptr write_inventories( return data; } +static chunk_inventories_map load_inventories(const ubyte* src, uint32_t size) { + chunk_inventories_map inventories; + ByteReader reader(src, size); + auto count = reader.getInt32(); + for (int i = 0; i < count; i++) { + uint index = reader.getInt32(); + uint size = reader.getInt32(); + auto map = json::from_binary(reader.pointer(), size); + reader.skip(size); + auto inv = std::make_shared(0, 0); + inv->deserialize(map.get()); + inventories[index] = inv; + } + return inventories; +} + void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { assert(chunk != nullptr); if (!chunk->flags.lighted) { @@ -150,7 +165,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { // Writing block inventories if (!chunk->inventories.empty()) { uint datasize; - auto data = write_inventories(chunk, datasize); + auto data = write_inventories(chunk->inventories, datasize); put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES, @@ -195,24 +210,25 @@ std::unique_ptr WorldRegions::getLights(int x, int z) { } chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { - chunk_inventories_map meta; uint32_t bytesSize; auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize); if (bytes == nullptr) { - return meta; + return {}; } - ByteReader reader(bytes, bytesSize); - auto count = reader.getInt32(); - for (int i = 0; i < count; i++) { - uint index = reader.getInt32(); - uint size = reader.getInt32(); - auto map = json::from_binary(reader.pointer(), size); - reader.skip(size); - auto inv = std::make_shared(0, 0); - inv->deserialize(map.get()); - meta[index] = inv; - } - return meta; + return load_inventories(bytes, bytesSize); +} + +void WorldRegions::processInventories( + int x, int z, const inventoryproc& func +) { + processRegion(x, z, REGION_LAYER_INVENTORIES, 0, + [=](std::unique_ptr data, uint32_t* size) { + auto inventories = load_inventories(data.get(), *size); + for (const auto& [_, inventory] : inventories) { + func(inventory.get()); + } + return write_inventories(inventories, *size); + }); } dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { @@ -231,8 +247,10 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { return map; } -void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - auto& layer = layers[REGION_LAYER_VOXELS]; +void WorldRegions::processRegion( + int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func +) { + auto& layer = layers[layerid]; if (layer.getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } @@ -250,25 +268,29 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { if (data == nullptr) { continue; } - data = compression::decompress( - data.get(), length, CHUNK_DATA_LEN, layer.compression - ); - if (func(data.get())) { - put(gx, - gz, - REGION_LAYER_VOXELS, - std::move(data), - CHUNK_DATA_LEN); + uint32_t totalLength = dataLen; + if (layer.compression != compression::Method::NONE) { + if (dataLen == 0) { + throw std::invalid_argument("invalid data length"); + } + data = compression::decompress( + data.get(), length, dataLen, layer.compression + ); + } else { + totalLength = length; + } + if (auto writeData = func(std::move(data), &totalLength)) { + put(gx, gz, layerid, std::move(writeData), totalLength); } } } } -fs::path WorldRegions::getRegionsFolder(int layer) const { - return layers[layer].folder; +const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const { + return layers[layerid].folder; } -void WorldRegions::write() { +void WorldRegions::writeAll() { for (auto& layer : layers) { fs::create_directories(layer.folder); layer.writeAll(); diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index eb2e803a..e4060002 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -16,6 +16,7 @@ #include "maths/voxmaths.hpp" #include "coders/compression.hpp" #include "files.hpp" +#include "world_regions_fwd.hpp" #define GLM_ENABLE_EXPERIMENTAL #include @@ -24,15 +25,6 @@ namespace fs = std::filesystem; inline constexpr uint REGION_HEADER_SIZE = 10; -enum RegionLayerIndex : uint { - REGION_LAYER_VOXELS = 0, - REGION_LAYER_LIGHTS, - REGION_LAYER_INVENTORIES, - REGION_LAYER_ENTITIES, - - REGION_LAYERS_COUNT -}; - inline constexpr uint REGION_SIZE_BIT = 5; inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); @@ -75,8 +67,10 @@ struct regfile { }; using regionsmap = std::unordered_map>; -using regionproc = std::function; +using regionproc = std::function(std::unique_ptr,uint32_t*)>; +using inventoryproc = std::function; +/// @brief Region file pointer keeping inUse flag on until destroyed class regfile_ptr { regfile* file; std::condition_variable* cv; @@ -209,6 +203,10 @@ public: size_t size ); + /// @brief Get chunk voxels data + /// @param x chunk.x + /// @param z chunk.z + /// @return voxels data buffer or nullptr std::unique_ptr getVoxels(int x, int z); /// @brief Get cached lights for chunk at x,z @@ -217,13 +215,30 @@ public: chunk_inventories_map fetchInventories(int x, int z); + /// @brief Load saved entities data for chunk + /// @param x chunk.x + /// @param z chunk.z + /// @return map with entities list as "data" dynamic::Map_sptr fetchEntities(int x, int z); - void processRegionVoxels(int x, int z, const regionproc& func); + /// @brief Load, process and save processed region chunks data + /// @param x region X + /// @param z region Z + /// @param layerid regions layer index + /// @param func processing callback + void processRegion( + int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func); - fs::path getRegionsFolder(int layer) const; + void processInventories( + int x, int z, const inventoryproc& func); - void write(); + /// @brief Get regions directory by layer index + /// @param layerid layer index + /// @return directory path + const fs::path& getRegionsFolder(RegionLayerIndex layerid) const; + + /// @brief Write all region layers + void writeAll(); /// @brief Extract X and Z from 'X_Z.bin' region file name. /// @param name source region file name diff --git a/src/files/world_regions_fwd.hpp b/src/files/world_regions_fwd.hpp new file mode 100644 index 00000000..e0398641 --- /dev/null +++ b/src/files/world_regions_fwd.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "typedefs.hpp" + +enum RegionLayerIndex : uint { + REGION_LAYER_VOXELS = 0, + REGION_LAYER_LIGHTS, + REGION_LAYER_INVENTORIES, + REGION_LAYER_ENTITIES, + + REGION_LAYERS_COUNT +}; diff --git a/src/items/Inventory.cpp b/src/items/Inventory.cpp index c2bc16ed..309ac57f 100644 --- a/src/items/Inventory.cpp +++ b/src/items/Inventory.cpp @@ -81,6 +81,15 @@ std::unique_ptr Inventory::serialize() const { return map; } +void Inventory::convert(const ContentReport* report) { + for (auto& slot : slots) { + itemid_t id = slot.getItemId(); + itemid_t replacement = report->items.getId(id); + slot.set(ItemStack(replacement, slot.getCount())); + } +} + +// TODO: remove void Inventory::convert(dynamic::Map* data, const ContentReport* report) { auto slotsarr = data->list("slots"); for (size_t i = 0; i < slotsarr->size(); i++) { diff --git a/src/items/Inventory.hpp b/src/items/Inventory.hpp index 260ec08c..0f80fb26 100644 --- a/src/items/Inventory.hpp +++ b/src/items/Inventory.hpp @@ -42,6 +42,7 @@ public: /* serializing inventory */ std::unique_ptr serialize() const override; + void convert(const ContentReport* report); static void convert(dynamic::Map* data, const ContentReport* report); inline void setId(int64_t id) { diff --git a/src/items/ItemStack.cpp b/src/items/ItemStack.cpp index 932f4417..99cb0a0e 100644 --- a/src/items/ItemStack.cpp +++ b/src/items/ItemStack.cpp @@ -16,6 +16,9 @@ void ItemStack::set(const ItemStack& item) { if (count == 0) { this->item = 0; } + if (this->item == 0) { + count = 0; + } } bool ItemStack::accepts(const ItemStack& other) const { diff --git a/src/logic/EngineController.cpp b/src/logic/EngineController.cpp index 91e77220..8cb53f28 100644 --- a/src/logic/EngineController.cpp +++ b/src/logic/EngineController.cpp @@ -59,6 +59,7 @@ std::shared_ptr create_converter( menu->setPage("main", false); engine->getGUI()->postRunnable([=]() { postRunnable(); }); }, + report->isUpgradeRequired(), true ); } From c15abfa7158b151ef1e1ed00dc7ae164542d5dde Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 3 Sep 2024 23:33:29 +0300 Subject: [PATCH 26/61] add util::Buffer, rle::encode16, rle::decode16 --- src/coders/rle.cpp | 39 +++++++++++++++++++++++++++ src/coders/rle.hpp | 3 +++ src/files/WorldConverter.cpp | 9 ++++--- src/files/WorldConverter.hpp | 4 +-- src/util/Buffer.hpp | 52 ++++++++++++++++++++++++++++++++++++ test/coders/rle.cpp | 43 +++++++++++++---------------- test/voxels/Chunk.cpp | 25 +++++++++++++++++ 7 files changed, 144 insertions(+), 31 deletions(-) create mode 100644 src/util/Buffer.hpp create mode 100644 test/voxels/Chunk.cpp diff --git a/src/coders/rle.cpp b/src/coders/rle.cpp index 771d1a41..a13454a5 100644 --- a/src/coders/rle.cpp +++ b/src/coders/rle.cpp @@ -35,6 +35,45 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) { return offset; } +size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) { + auto src16 = reinterpret_cast(src); + auto dst16 = reinterpret_cast(dst); + size_t offset = 0; + for (size_t i = 0; i < srclen / 2;) { + uint16_t len = src16[i++]; + uint16_t c = src16[i++]; + for (size_t j = 0; j <= len; j++) { + dst16[offset++] = c; + } + } + return offset * 2; +} + +size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) { + if (srclen == 0) { + return 0; + } + auto src16 = reinterpret_cast(src); + auto dst16 = reinterpret_cast(dst); + size_t offset = 0; + uint16_t counter = 0; + uint16_t c = src16[0]; + for (size_t i = 1; i < srclen / 2; i++) { + uint16_t cnext = src16[i]; + if (cnext != c || counter == 0xFFFF) { + dst16[offset++] = counter; + dst16[offset++] = c; + c = cnext; + counter = 0; + } else { + counter++; + } + } + dst16[offset++] = counter; + dst16[offset++] = c; + return offset * 2; +} + size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) { size_t offset = 0; for (size_t i = 0; i < srclen;) { diff --git a/src/coders/rle.hpp b/src/coders/rle.hpp index 0692cc9a..eb6c5b1d 100644 --- a/src/coders/rle.hpp +++ b/src/coders/rle.hpp @@ -5,6 +5,9 @@ namespace rle { size_t encode(const ubyte* src, size_t length, ubyte* dst); size_t decode(const ubyte* src, size_t length, ubyte* dst); + + size_t encode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst); } namespace extrle { diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index a6b977bb..ee3f1fa6 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -61,7 +61,7 @@ void WorldConverter::createUpgradeTasks() { if (issue.regionLayer == REGION_LAYER_VOXELS) { addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS); } else { - addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_SIMPLE); + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION); } } } @@ -159,7 +159,7 @@ std::shared_ptr WorldConverter::startTask( return pool; } -void WorldConverter::upgradeSimple(const fs::path& file, int x, int z) const { +void WorldConverter::upgradeRegion(const fs::path& file, int x, int z) const { throw std::runtime_error("unsupported region format"); } @@ -194,10 +194,11 @@ void WorldConverter::convert(const ConvertTask& task) const { if (!fs::is_regular_file(task.file)) return; switch (task.type) { - case ConvertTaskType::UPGRADE_SIMPLE: - upgradeSimple(task.file, task.x, task.z); + case ConvertTaskType::UPGRADE_REGION: + upgradeRegion(task.file, task.x, task.z); break; case ConvertTaskType::UPGRADE_VOXELS: + upgradeRegion(task.file, task.x, task.z); upgradeVoxels(task.file, task.x, task.z); break; case ConvertTaskType::VOXELS: diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index d8a3ce5f..63cf5993 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -23,7 +23,7 @@ enum class ConvertTaskType { /// @brief rewrite player PLAYER, /// @brief refresh region file version - UPGRADE_SIMPLE, + UPGRADE_REGION, /// @brief rewrite voxels region file to new format UPGRADE_VOXELS, }; @@ -45,7 +45,7 @@ class WorldConverter : public Task { uint tasksDone = 0; bool upgradeMode; - void upgradeSimple(const fs::path& file, int x, int z) const; + void upgradeRegion(const fs::path& file, int x, int z) const; void upgradeVoxels(const fs::path& file, int x, int z) const; void convertPlayer(const fs::path& file) const; void convertVoxels(const fs::path& file, int x, int z) const; diff --git a/src/util/Buffer.hpp b/src/util/Buffer.hpp new file mode 100644 index 00000000..7ff0ca8a --- /dev/null +++ b/src/util/Buffer.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +namespace util { + template + class Buffer { + std::unique_ptr ptr; + size_t length; + public: + Buffer(size_t length) + : ptr(std::make_unique(length)), length(length) { + } + + Buffer(std::unique_ptr ptr, size_t length) + : ptr(std::move(ptr)), length(length) {} + + Buffer(const T* src, size_t length) + : ptr(std::make_unique(length)), length(length) { + std::memcpy(ptr, src, length); + } + + T& operator[](long long index) { + return ptr[index]; + } + + const T& operator[](long long index) const { + return ptr[index]; + } + + T* data() { + return ptr.get(); + } + + const T* data() const { + return ptr.get(); + } + + size_t size() const { + return length; + } + + std::unique_ptr release() { + return std::move(ptr); + } + + Buffer clone() const { + return Buffer(ptr.get(), length); + } + }; +} diff --git a/test/coders/rle.cpp b/test/coders/rle.cpp index c93a034a..4ce06d1a 100644 --- a/test/coders/rle.cpp +++ b/test/coders/rle.cpp @@ -3,46 +3,39 @@ #include "typedefs.hpp" #include "coders/rle.hpp" -TEST(RLE, EncodeDecode) { - const int initial_size = 50'000; +static void test_encode_decode( + size_t(*encodefunc)(const ubyte*, size_t, ubyte*), + size_t(*decodefunc)(const ubyte*, size_t, ubyte*) +) { + const size_t initial_size = 50'000; uint8_t initial[initial_size]; uint8_t next = rand(); - for (int i = 0; i < initial_size; i++) { + for (size_t i = 0; i < initial_size; i++) { initial[i] = next; if (rand() % 13 == 0) { next = rand(); } } uint8_t encoded[initial_size * 2]; - auto encoded_size = rle::encode(initial, initial_size, encoded); + size_t encoded_size = encodefunc(initial, initial_size, encoded); uint8_t decoded[initial_size * 2]; - auto decoded_size = rle::decode(encoded, encoded_size, decoded); + size_t decoded_size = decodefunc(encoded, encoded_size, decoded); EXPECT_EQ(decoded_size, initial_size); - for (int i = 0; i < decoded_size; i++) { + for (size_t i = 0; i < decoded_size; i++) { EXPECT_EQ(decoded[i], initial[i]); } } +TEST(RLE, EncodeDecode) { + test_encode_decode(rle::encode, rle::decode); +} + +TEST(RLE16, EncodeDecode) { + test_encode_decode(rle::encode16, rle::decode16); +} + TEST(ExtRLE, EncodeDecode) { - const int initial_size = 50'000; - uint8_t initial[initial_size]; - uint8_t next = rand(); - for (int i = 0; i < initial_size; i++) { - initial[i] = next; - if (rand() % 13 == 0) { - next = rand(); - } - } - uint8_t encoded[initial_size * 2]; - auto encoded_size = extrle::encode(initial, initial_size, encoded); - uint8_t decoded[initial_size * 2]; - auto decoded_size = extrle::decode(encoded, encoded_size, decoded); - - EXPECT_EQ(decoded_size, initial_size); - - for (int i = 0; i < decoded_size; i++) { - EXPECT_EQ(decoded[i], initial[i]); - } + test_encode_decode(extrle::encode, extrle::decode); } diff --git a/test/voxels/Chunk.cpp b/test/voxels/Chunk.cpp new file mode 100644 index 00000000..2a82882b --- /dev/null +++ b/test/voxels/Chunk.cpp @@ -0,0 +1,25 @@ +#include + +#include "voxels/Chunk.hpp" + +TEST(Chunk, EncodeDecode) { + Chunk chunk1(0, 0); + for (uint i = 0; i < CHUNK_VOL; i++) { + chunk1.voxels[i].id = rand(); + chunk1.voxels[i].state.rotation = rand(); + chunk1.voxels[i].state.segment = rand(); + chunk1.voxels[i].state.userbits = rand(); + } + auto bytes = chunk1.encode(); + + Chunk chunk2(0, 0); + chunk2.decode(bytes.get()); + + for (uint i = 0; i < CHUNK_VOL; i++) { + EXPECT_EQ(chunk1.voxels[i].id, chunk2.voxels[i].id); + EXPECT_EQ( + blockstate2int(chunk1.voxels[i].state), + blockstate2int(chunk2.voxels[i].state) + ); + } +} From ea3d8b2c0849d94e0a6dde09bf25f9ca07b8698a Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 3 Sep 2024 23:45:31 +0300 Subject: [PATCH 27/61] add region file format specification --- {src/coders => doc/specs}/binary_json_spec.md | 6 --- doc/specs/region_file_v2_spec.md | 41 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) rename {src/coders => doc/specs}/binary_json_spec.md (92%) create mode 100644 doc/specs/region_file_v2_spec.md diff --git a/src/coders/binary_json_spec.md b/doc/specs/binary_json_spec.md similarity index 92% rename from src/coders/binary_json_spec.md rename to doc/specs/binary_json_spec.md index e7b6f450..32158dd5 100644 --- a/src/coders/binary_json_spec.md +++ b/doc/specs/binary_json_spec.md @@ -49,9 +49,3 @@ int32 = 4byte int16 = 2byte byte = %x00-FF ``` - -## VoxelEngine format support - -Current implementation does not support types: bytes array, null, compressed document. - -All unsupported types will be implemented in future. diff --git a/doc/specs/region_file_v2_spec.md b/doc/specs/region_file_v2_spec.md new file mode 100644 index 00000000..4e4d2890 --- /dev/null +++ b/doc/specs/region_file_v2_spec.md @@ -0,0 +1,41 @@ +# Region File (version 2) + +File format BNF (RFC 5234): + +```bnf +file = header (*chunk) offsets complete file +header = magic %x02 %x00 magic number, version and reserved + zero byte + +magic = %x2E %x56 %x4F %x58 '.VOXREG\0' + %x52 %x45 %x47 %x00 + +chunk = int32 (*byte) byte array with size prefix +offsets = (1024*int32) offsets table +int32 = 4byte signed big-endian 32 bit integer +byte = %x00-FF 8 bit unsigned integer +``` + +C struct visualization: + +```c +typedef unsigned char byte; + +struct file { + // 10 bytes + struct { + char magic[8] = ".VOXREG"; + byte version = 2; + byte reserved = 0; + } header; + + struct { + int32_t size; // byteorder: big-endian + byte* data; + } chunks[1024]; // file does not contain zero sizes for missing chunks + + int32_t offsets[1024]; // byteorder: big-endian +}; +``` + +Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size). From 3d3679c8b1858bfe5e89364e019f23eefc73907d Mon Sep 17 00:00:00 2001 From: MihailRis Date: Wed, 4 Sep 2024 00:10:39 +0300 Subject: [PATCH 28/61] add voxel regions layer chunk structure specification --- ...on_file_v2_spec.md => region_file_spec.md} | 0 doc/specs/region_voxels_chunk_spec.md | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+) rename doc/specs/{region_file_v2_spec.md => region_file_spec.md} (100%) create mode 100644 doc/specs/region_voxels_chunk_spec.md diff --git a/doc/specs/region_file_v2_spec.md b/doc/specs/region_file_spec.md similarity index 100% rename from doc/specs/region_file_v2_spec.md rename to doc/specs/region_file_spec.md diff --git a/doc/specs/region_voxels_chunk_spec.md b/doc/specs/region_voxels_chunk_spec.md new file mode 100644 index 00000000..7467944c --- /dev/null +++ b/doc/specs/region_voxels_chunk_spec.md @@ -0,0 +1,26 @@ +# Voxels Chunk (version 1) + +Voxel regions layer chunk structure. + +Values are separated for extRLE8 compression efficiency. + +File format BNF (RFC 5234): + +```bnf +chunk = (65536*byte) block indices (most significant bytes) + (65536*byte) block indices (least significant bytes) + (65536*byte) block states (most significant bytes) + (65536*byte) block states (least significant bytes) + +byte = %x00-FF 8 bit unsigned integer +``` + +65536 is number of voxels per chunk (16\*256\*16) + +## Block state + +Block state is encoded in 16 bits: +- 0-2 bits (3) - block rotation index +- 3-5 bits (3) - segment block bits +- 6-7 bits (2) - reserved +- 8-15 bits (8) - user bits From 0b3bb36188bc6bffdfaafacafaa81e31eac65125 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Wed, 4 Sep 2024 15:41:51 +0300 Subject: [PATCH 29/61] update byte_utils --- src/coders/byte_utils.cpp | 90 ++++++++++++++++---------------------- src/coders/byte_utils.hpp | 26 +++++------ src/util/Buffer.hpp | 2 +- src/util/data_io.hpp | 18 ++++++++ test/coders/byte_utils.cpp | 20 +++++++++ 5 files changed, 88 insertions(+), 68 deletions(-) create mode 100644 test/coders/byte_utils.cpp diff --git a/src/coders/byte_utils.cpp b/src/coders/byte_utils.cpp index d28e31ce..50c08563 100644 --- a/src/coders/byte_utils.cpp +++ b/src/coders/byte_utils.cpp @@ -4,6 +4,8 @@ #include #include +#include "util/data_io.hpp" + void ByteBuilder::put(ubyte b) { buffer.push_back(b); } @@ -30,28 +32,24 @@ void ByteBuilder::put(const ubyte* arr, size_t size) { } void ByteBuilder::putInt16(int16_t val) { - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int16_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int16_t)); } void ByteBuilder::putInt32(int32_t val) { - buffer.reserve(buffer.size() + 4); - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); - buffer.push_back(static_cast(val >> 16 & 255)); - buffer.push_back(static_cast(val >> 24 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int32_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int32_t)); } void ByteBuilder::putInt64(int64_t val) { - buffer.reserve(buffer.size() + 8); - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); - buffer.push_back(static_cast(val >> 16 & 255)); - buffer.push_back(static_cast(val >> 24 & 255)); - buffer.push_back(static_cast(val >> 32 & 255)); - buffer.push_back(static_cast(val >> 40 & 255)); - buffer.push_back(static_cast(val >> 48 & 255)); - buffer.push_back(static_cast(val >> 56 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int64_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int64_t)); } void ByteBuilder::putFloat32(float val) { @@ -71,27 +69,18 @@ void ByteBuilder::set(size_t position, ubyte val) { } void ByteBuilder::setInt16(size_t position, int16_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position] = val >> 8 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int16_t)); } void ByteBuilder::setInt32(size_t position, int32_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position++] = val >> 8 & 255; - buffer[position++] = val >> 16 & 255; - buffer[position] = val >> 24 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int32_t)); } void ByteBuilder::setInt64(size_t position, int64_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position++] = val >> 8 & 255; - buffer[position++] = val >> 16 & 255; - buffer[position++] = val >> 24 & 255; - - buffer[position++] = val >> 32 & 255; - buffer[position++] = val >> 40 & 255; - buffer[position++] = val >> 48 & 255; - buffer[position] = val >> 56 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int64_t)); } std::vector ByteBuilder::build() { @@ -111,7 +100,7 @@ void ByteReader::checkMagic(const char* data, size_t size) { throw std::runtime_error("invalid magic number"); } for (size_t i = 0; i < size; i++) { - if (this->data[pos + i] != (ubyte)data[i]) { + if (this->data[pos + i] != static_cast(data[i])) { throw std::runtime_error("invalid magic number"); } } @@ -133,38 +122,33 @@ ubyte ByteReader::peek() { } int16_t ByteReader::getInt16() { - if (pos + 2 > size) { + if (pos + sizeof(int16_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 2; - return (static_cast(data[pos - 1]) << 8) | - (static_cast(data[pos - 2])); + int16_t value; + std::memcpy(&value, data + pos, sizeof(int16_t)); + pos += sizeof(int16_t); + return dataio::le2h(value); } int32_t ByteReader::getInt32() { - if (pos + 4 > size) { + if (pos + sizeof(int32_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 4; - return (static_cast(data[pos - 1]) << 24) | - (static_cast(data[pos - 2]) << 16) | - (static_cast(data[pos - 3]) << 8) | - (static_cast(data[pos - 4])); + int32_t value; + std::memcpy(&value, data + pos, sizeof(int32_t)); + pos += sizeof(int32_t); + return dataio::le2h(value); } int64_t ByteReader::getInt64() { - if (pos + 8 > size) { + if (pos + sizeof(int64_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 8; - return (static_cast(data[pos - 1]) << 56) | - (static_cast(data[pos - 2]) << 48) | - (static_cast(data[pos - 3]) << 40) | - (static_cast(data[pos - 4]) << 32) | - (static_cast(data[pos - 5]) << 24) | - (static_cast(data[pos - 6]) << 16) | - (static_cast(data[pos - 7]) << 8) | - (static_cast(data[pos - 8])); + int64_t value; + std::memcpy(&value, data + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + return dataio::le2h(value); } float ByteReader::getFloat32() { @@ -183,7 +167,7 @@ double ByteReader::getFloat64() { const char* ByteReader::getCString() { const char* cstr = reinterpret_cast(data + pos); - pos += strlen(cstr) + 1; + pos += std::strlen(cstr) + 1; return cstr; } diff --git a/src/coders/byte_utils.hpp b/src/coders/byte_utils.hpp index 20c5d150..855ebaec 100644 --- a/src/coders/byte_utils.hpp +++ b/src/coders/byte_utils.hpp @@ -5,28 +5,27 @@ #include "typedefs.hpp" -/* byteorder: little-endian */ class ByteBuilder { std::vector buffer; public: - /* Write one byte (8 bit unsigned integer) */ + /// @brief Write one byte (8 bit unsigned integer) void put(ubyte b); - /* Write c-string (bytes array terminated with '\00') */ + /// @brief Write c-string (bytes array terminated with '\00') void putCStr(const char* str); - /* Write signed 16 bit integer */ + /// @brief Write signed 16 bit little-endian integer void putInt16(int16_t val); - /* Write signed 32 bit integer */ + /// @brief Write signed 32 bit integer void putInt32(int32_t val); - /* Write signed 64 bit integer */ + /// @brief Write signed 64 bit integer void putInt64(int64_t val); - /* Write 32 bit floating-point number */ + /// @brief Write 32 bit floating-point number void putFloat32(float val); - /* Write 64 bit floating-point number */ + /// @brief Write 64 bit floating-point number void putFloat64(double val); - /* Write string (uint32 length + bytes) */ + /// @brief Write string (uint32 length + bytes) void put(const std::string& s); - /* Write sequence of bytes without any header */ + /// @brief Write sequence of bytes without any header void put(const ubyte* arr, size_t size); void set(size_t position, ubyte val); @@ -44,7 +43,6 @@ public: std::vector build(); }; -/// byteorder: little-endian class ByteReader { const ubyte* data; size_t size; @@ -58,11 +56,11 @@ public: ubyte get(); /// @brief Read one byte (unsigned 8 bit integer) without pointer move ubyte peek(); - /// @brief Read signed 16 bit integer + /// @brief Read signed 16 bit little-endian integer int16_t getInt16(); - /// @brief Read signed 32 bit integer + /// @brief Read signed 32 bit little-endian integer int32_t getInt32(); - /// @brief Read signed 64 bit integer + /// @brief Read signed 64 bit little-endian integer int64_t getInt64(); /// @brief Read 32 bit floating-point number float getFloat32(); diff --git a/src/util/Buffer.hpp b/src/util/Buffer.hpp index 7ff0ca8a..4787d2ed 100644 --- a/src/util/Buffer.hpp +++ b/src/util/Buffer.hpp @@ -18,7 +18,7 @@ namespace util { Buffer(const T* src, size_t length) : ptr(std::make_unique(length)), length(length) { - std::memcpy(ptr, src, length); + std::memcpy(ptr.get(), src, length); } T& operator[](long long index) { diff --git a/src/util/data_io.hpp b/src/util/data_io.hpp index 93022f7d..4d2d69ab 100644 --- a/src/util/data_io.hpp +++ b/src/util/data_io.hpp @@ -41,6 +41,15 @@ namespace dataio { } } + /// @brief Convert hardware byte-order to big-endian + /// @tparam T value type + /// @param value source integer + /// @return big-endian integer + template + T h2be(T value){ + return be2h(value); + } + /// @brief Convert little-endian to hardware byte-order /// @tparam T value type /// @param value source integer @@ -54,6 +63,15 @@ namespace dataio { } } + /// @brief Convert hardware byte-order to little-endian + /// @tparam T value type + /// @param value source integer + /// @return little-endian integer + template + T h2le(T value){ + return le2h(value); + } + /// @brief Read big-endian 16 bit signed integer from bytes inline int16_t read_int16_big(const ubyte* src, size_t offset) { return (src[offset] << 8) | (src[offset + 1]); diff --git a/test/coders/byte_utils.cpp b/test/coders/byte_utils.cpp new file mode 100644 index 00000000..6b0be61d --- /dev/null +++ b/test/coders/byte_utils.cpp @@ -0,0 +1,20 @@ +#include + +#include "coders/byte_utils.hpp" + +TEST(byte_utils, BytesBuildAndRead) { + ByteBuilder builder; + builder.putCStr("MAGIC.#"); + builder.put(50); + builder.putInt16(1980); + builder.putInt32(123456789); + builder.putInt64(98765432123456789LL); + auto data = builder.build(); + + ByteReader reader(data.data(), data.size()); + reader.checkMagic("MAGIC.#", 8); + EXPECT_EQ(reader.get(), 50); + EXPECT_EQ(reader.getInt16(), 1980); + EXPECT_EQ(reader.getInt32(), 123456789); + EXPECT_EQ(reader.getInt64(), 98765432123456789LL); +} From 73a8343f610933edede93bd5c4be8c936150463c Mon Sep 17 00:00:00 2001 From: MihailRis Date: Wed, 4 Sep 2024 23:27:52 +0300 Subject: [PATCH 30/61] add extRLE16 encoder/decoder --- src/coders/rle.cpp | 83 ++++++++++++++++++++++++++++++++++++++++---- src/coders/rle.hpp | 4 +++ src/util/data_io.hpp | 2 +- test/coders/rle.cpp | 4 +++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/coders/rle.cpp b/src/coders/rle.cpp index a13454a5..f049492e 100644 --- a/src/coders/rle.cpp +++ b/src/coders/rle.cpp @@ -1,5 +1,7 @@ #include "rle.hpp" +#include "util/data_io.hpp" + size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) { size_t offset = 0; for (size_t i = 0; i < srclen;) { @@ -40,8 +42,8 @@ size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) { auto dst16 = reinterpret_cast(dst); size_t offset = 0; for (size_t i = 0; i < srclen / 2;) { - uint16_t len = src16[i++]; - uint16_t c = src16[i++]; + uint16_t len = dataio::le2h(src16[i++]); + uint16_t c = dataio::le2h(src16[i++]); for (size_t j = 0; j <= len; j++) { dst16[offset++] = c; } @@ -61,16 +63,16 @@ size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) { for (size_t i = 1; i < srclen / 2; i++) { uint16_t cnext = src16[i]; if (cnext != c || counter == 0xFFFF) { - dst16[offset++] = counter; - dst16[offset++] = c; + dst16[offset++] = dataio::h2le(counter); + dst16[offset++] = dataio::h2le(c); c = cnext; counter = 0; } else { counter++; } } - dst16[offset++] = counter; - dst16[offset++] = c; + dst16[offset++] = dataio::h2le(counter); + dst16[offset++] = dataio::h2le(c); return offset * 2; } @@ -80,7 +82,7 @@ size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) { uint len = src[i++]; if (len & 0x80) { len &= 0x7F; - len |= ((uint)src[i++]) << 7; + len |= (static_cast(src[i++])) << 7; } ubyte c = src[i++]; for (size_t j = 0; j <= len; j++) { @@ -122,3 +124,70 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) { dst[offset++] = c; return offset; } + +size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) { + auto dst = reinterpret_cast(dst8); + size_t offset = 0; + for (size_t i = 0; i < srclen;) { + uint len = src[i++]; + bool widechar = len & 0x40; + if (len & 0x80) { + len &= 0x3F; + len |= (static_cast(src[i++])) << 6; + } else { + len &= 0x3F; + } + uint16_t c = src[i++]; + if (widechar) { + c |= ((static_cast(src[i++])) << 8); + } + for (size_t j = 0; j <= len; j++) { + dst[offset++] = c; + } + } + return offset * 2; +} + +size_t extrle::encode16(const ubyte* src8, size_t srclen, ubyte* dst) { + if (srclen == 0) { + return 0; + } + auto src = reinterpret_cast(src8); + size_t offset = 0; + uint counter = 0; + uint16_t c = src[0]; + for (size_t i = 1; i < srclen/2; i++) { + uint16_t cnext = src[i]; + if (cnext != c || counter == max_sequence) { + if (counter >= 0x40) { + dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F); + dst[offset++] = counter >> 6; + } else { + dst[offset++] = counter | ((c > 255) << 6); + } + if (c > 255) { + dst[offset++] = c & 0xFF; + dst[offset++] = c >> 8; + } else { + dst[offset++] = c; + } + c = cnext; + counter = 0; + } else { + counter++; + } + } + if (counter >= 0x40) { + dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F); + dst[offset++] = counter >> 6; + } else { + dst[offset++] = counter | ((c > 255) << 6); + } + if (c > 255) { + dst[offset++] = c & 0xFF; + dst[offset++] = c >> 8; + } else { + dst[offset++] = c; + } + return offset; +} diff --git a/src/coders/rle.hpp b/src/coders/rle.hpp index eb6c5b1d..c087ac69 100644 --- a/src/coders/rle.hpp +++ b/src/coders/rle.hpp @@ -14,4 +14,8 @@ namespace extrle { constexpr uint max_sequence = 0x7FFF; size_t encode(const ubyte* src, size_t length, ubyte* dst); size_t decode(const ubyte* src, size_t length, ubyte* dst); + + constexpr uint max_sequence16 = 0x3FFF; + size_t encode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst); } diff --git a/src/util/data_io.hpp b/src/util/data_io.hpp index 4d2d69ab..abf5bfa2 100644 --- a/src/util/data_io.hpp +++ b/src/util/data_io.hpp @@ -33,7 +33,7 @@ namespace dataio { /// @param value source integer /// @return integer with hardware byte-order template - T be2h(T value){ + inline T be2h(T value){ if (is_big_endian()) { return value; } else { diff --git a/test/coders/rle.cpp b/test/coders/rle.cpp index 4ce06d1a..3790fdb1 100644 --- a/test/coders/rle.cpp +++ b/test/coders/rle.cpp @@ -39,3 +39,7 @@ TEST(RLE16, EncodeDecode) { TEST(ExtRLE, EncodeDecode) { test_encode_decode(extrle::encode, extrle::decode); } + +TEST(ExtRLE16, EncodeDecode) { + test_encode_decode(extrle::encode16, extrle::decode16); +} From 184e9c66484854a638bd67abeaa267c687116fac Mon Sep 17 00:00:00 2001 From: MihailRis Date: Wed, 4 Sep 2024 23:37:39 +0300 Subject: [PATCH 31/61] update region file format 2 to 3 (WIP) --- doc/specs/outdated/region_file_spec_v2.md | 41 +++++++ .../outdated/region_voxels_chunk_spec_v1.md | 26 +++++ doc/specs/region_file_spec.md | 29 +++-- doc/specs/region_voxels_chunk_spec.md | 13 +-- src/coders/compression.cpp | 48 +++++--- src/coders/compression.hpp | 2 +- src/files/RegionsLayer.cpp | 4 + src/files/WorldConverter.cpp | 28 ++--- src/files/WorldConverter.hpp | 7 +- src/files/WorldRegions.cpp | 4 + src/files/WorldRegions.hpp | 4 + src/files/compatibility.cpp | 109 ++++++++++++++++++ src/files/compatibility.hpp | 14 +++ src/files/files.cpp | 6 + src/files/files.hpp | 2 + test/files/compatibility.cpp | 15 +++ 16 files changed, 295 insertions(+), 57 deletions(-) create mode 100644 doc/specs/outdated/region_file_spec_v2.md create mode 100644 doc/specs/outdated/region_voxels_chunk_spec_v1.md create mode 100644 src/files/compatibility.cpp create mode 100644 src/files/compatibility.hpp create mode 100644 test/files/compatibility.cpp diff --git a/doc/specs/outdated/region_file_spec_v2.md b/doc/specs/outdated/region_file_spec_v2.md new file mode 100644 index 00000000..4e4d2890 --- /dev/null +++ b/doc/specs/outdated/region_file_spec_v2.md @@ -0,0 +1,41 @@ +# Region File (version 2) + +File format BNF (RFC 5234): + +```bnf +file = header (*chunk) offsets complete file +header = magic %x02 %x00 magic number, version and reserved + zero byte + +magic = %x2E %x56 %x4F %x58 '.VOXREG\0' + %x52 %x45 %x47 %x00 + +chunk = int32 (*byte) byte array with size prefix +offsets = (1024*int32) offsets table +int32 = 4byte signed big-endian 32 bit integer +byte = %x00-FF 8 bit unsigned integer +``` + +C struct visualization: + +```c +typedef unsigned char byte; + +struct file { + // 10 bytes + struct { + char magic[8] = ".VOXREG"; + byte version = 2; + byte reserved = 0; + } header; + + struct { + int32_t size; // byteorder: big-endian + byte* data; + } chunks[1024]; // file does not contain zero sizes for missing chunks + + int32_t offsets[1024]; // byteorder: big-endian +}; +``` + +Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size). diff --git a/doc/specs/outdated/region_voxels_chunk_spec_v1.md b/doc/specs/outdated/region_voxels_chunk_spec_v1.md new file mode 100644 index 00000000..7467944c --- /dev/null +++ b/doc/specs/outdated/region_voxels_chunk_spec_v1.md @@ -0,0 +1,26 @@ +# Voxels Chunk (version 1) + +Voxel regions layer chunk structure. + +Values are separated for extRLE8 compression efficiency. + +File format BNF (RFC 5234): + +```bnf +chunk = (65536*byte) block indices (most significant bytes) + (65536*byte) block indices (least significant bytes) + (65536*byte) block states (most significant bytes) + (65536*byte) block states (least significant bytes) + +byte = %x00-FF 8 bit unsigned integer +``` + +65536 is number of voxels per chunk (16\*256\*16) + +## Block state + +Block state is encoded in 16 bits: +- 0-2 bits (3) - block rotation index +- 3-5 bits (3) - segment block bits +- 6-7 bits (2) - reserved +- 8-15 bits (8) - user bits diff --git a/doc/specs/region_file_spec.md b/doc/specs/region_file_spec.md index 4e4d2890..30601798 100644 --- a/doc/specs/region_file_spec.md +++ b/doc/specs/region_file_spec.md @@ -1,18 +1,21 @@ -# Region File (version 2) +# Region File (version 3) File format BNF (RFC 5234): ```bnf file = header (*chunk) offsets complete file -header = magic %x02 %x00 magic number, version and reserved - zero byte +header = magic %x02 byte magic number, version and compression + method magic = %x2E %x56 %x4F %x58 '.VOXREG\0' %x52 %x45 %x47 %x00 -chunk = int32 (*byte) byte array with size prefix -offsets = (1024*int32) offsets table -int32 = 4byte signed big-endian 32 bit integer +chunk = uint32 uint32 (*byte) byte array with size and source size + prefix where source size is + decompressed chunk data size + +offsets = (1024*uint32) offsets table +int32 = 4byte unsigned big-endian 32 bit integer byte = %x00-FF 8 bit unsigned integer ``` @@ -25,17 +28,23 @@ struct file { // 10 bytes struct { char magic[8] = ".VOXREG"; - byte version = 2; - byte reserved = 0; + byte version = 3; + byte compression; } header; struct { - int32_t size; // byteorder: big-endian + uint32_t size; // byteorder: little-endian + uint32_t sourceSize; // byteorder: little-endian byte* data; } chunks[1024]; // file does not contain zero sizes for missing chunks - int32_t offsets[1024]; // byteorder: big-endian + uint32_t offsets[1024]; // byteorder: little-endian }; ``` Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size). + +Available compression methods: +0. no compression +1. extRLE8 +2. extRLE16 diff --git a/doc/specs/region_voxels_chunk_spec.md b/doc/specs/region_voxels_chunk_spec.md index 7467944c..d43208cb 100644 --- a/doc/specs/region_voxels_chunk_spec.md +++ b/doc/specs/region_voxels_chunk_spec.md @@ -1,17 +1,14 @@ -# Voxels Chunk (version 1) +# Voxels Chunk (version 2) -Voxel regions layer chunk structure. - -Values are separated for extRLE8 compression efficiency. +IDs and states are separated for extRLE16 compression efficiency. File format BNF (RFC 5234): ```bnf -chunk = (65536*byte) block indices (most significant bytes) - (65536*byte) block indices (least significant bytes) - (65536*byte) block states (most significant bytes) - (65536*byte) block states (least significant bytes) +chunk = (65536*uint16) block ids + (65536*uint16) block states +uint16 = 2byte 16 bit little-endian unsigned integer byte = %x00-FF 8 bit unsigned integer ``` diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp index 343328cf..74f68528 100644 --- a/src/coders/compression.cpp +++ b/src/coders/compression.cpp @@ -25,29 +25,38 @@ static std::shared_ptr get_buffer(size_t minSize) { return nullptr; } +static auto compress_rle( + const ubyte* src, + size_t srclen, + size_t& len, + size_t(*encodefunc)(const ubyte*, size_t, ubyte*) +) { + auto buffer = get_buffer(srclen * 2); + auto bytes = buffer.get(); + std::unique_ptr uptr; + if (bytes == nullptr) { + uptr = std::make_unique(srclen * 2); + bytes = uptr.get(); + } + len = encodefunc(src, srclen, bytes); + if (uptr) { + return uptr; + } + auto data = std::make_unique(len); + std::memcpy(data.get(), bytes, len); + return data; +} + std::unique_ptr compression::compress( const ubyte* src, size_t srclen, size_t& len, Method method ) { switch (method) { case Method::NONE: throw std::invalid_argument("compression method is NONE"); - case Method::EXTRLE8: { - // max extrle out size is srcLen * 2 - auto buffer = get_buffer(srclen * 2); - auto bytes = buffer.get(); - std::unique_ptr uptr; - if (bytes == nullptr) { - uptr = std::make_unique(srclen * 2); - bytes = uptr.get(); - } - len = extrle::encode(src, srclen, bytes); - if (uptr) { - return uptr; - } - auto data = std::make_unique(len); - std::memcpy(data.get(), bytes, len); - return data; - } + case Method::EXTRLE8: + return compress_rle(src, srclen, len, extrle::encode); + case Method::EXTRLE16: + return compress_rle(src, srclen, len, extrle::encode16); case Method::GZIP: { auto buffer = gzip::compress(src, srclen); auto data = std::make_unique(buffer.size()); @@ -71,6 +80,11 @@ std::unique_ptr compression::decompress( extrle::decode(src, srclen, decompressed.get()); return decompressed; } + case Method::EXTRLE16: { + auto decompressed = std::make_unique(dstlen); + extrle::decode16(src, srclen, decompressed.get()); + return decompressed; + } case Method::GZIP: { auto buffer = gzip::decompress(src, srclen); if (buffer.size() != dstlen) { diff --git a/src/coders/compression.hpp b/src/coders/compression.hpp index 09f50b2a..98b7a0e3 100644 --- a/src/coders/compression.hpp +++ b/src/coders/compression.hpp @@ -6,7 +6,7 @@ namespace compression { enum class Method { - NONE, EXTRLE8, GZIP + NONE, EXTRLE8, EXTRLE16, GZIP }; /// @brief Compress buffer diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp index 2ec8782a..3f0101f4 100644 --- a/src/files/RegionsLayer.cpp +++ b/src/files/RegionsLayer.cpp @@ -135,6 +135,10 @@ WorldRegion* RegionsLayer::getRegion(int x, int z) { return found->second.get(); } +fs::path RegionsLayer::getRegionFilePath(int x, int z) const { + return folder / get_region_filename(x, z); +} + WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) { if (auto region = getRegion(x, z)) { return region; diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index ee3f1fa6..4748172d 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -6,6 +6,7 @@ #include #include "content/ContentReport.hpp" +#include "files/compatibility.hpp" #include "data/dynamic.hpp" #include "debug/Logger.hpp" #include "files/files.hpp" @@ -48,7 +49,7 @@ void WorldConverter::addRegionsTasks( logger.error() << "could not parse region name " << name; continue; } - tasks.push(ConvertTask {taskType, file.path(), x, z}); + tasks.push(ConvertTask {taskType, file.path(), x, z, layerid}); } } @@ -58,11 +59,7 @@ void WorldConverter::createUpgradeTasks() { if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) { continue; } - if (issue.regionLayer == REGION_LAYER_VOXELS) { - addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS); - } else { - addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION); - } + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION); } } @@ -159,12 +156,13 @@ std::shared_ptr WorldConverter::startTask( return pool; } -void WorldConverter::upgradeRegion(const fs::path& file, int x, int z) const { - throw std::runtime_error("unsupported region format"); -} - -void WorldConverter::upgradeVoxels(const fs::path& file, int x, int z) const { - throw std::runtime_error("unsupported region format"); +void WorldConverter::upgradeRegion( + const fs::path& file, int x, int z, RegionLayerIndex layer +) const { + auto path = wfile->getRegions().getRegionFilePath(layer, x, z); + auto bytes = files::read_bytes_buffer(path); + auto buffer = compatibility::convertRegion2to3(bytes, layer); + files::write_bytes(path, buffer.data(), buffer.size()); } void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const { @@ -195,11 +193,7 @@ void WorldConverter::convert(const ConvertTask& task) const { switch (task.type) { case ConvertTaskType::UPGRADE_REGION: - upgradeRegion(task.file, task.x, task.z); - break; - case ConvertTaskType::UPGRADE_VOXELS: - upgradeRegion(task.file, task.x, task.z); - upgradeVoxels(task.file, task.x, task.z); + upgradeRegion(task.file, task.x, task.z, task.layer); break; case ConvertTaskType::VOXELS: convertVoxels(task.file, task.x, task.z); diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index 63cf5993..fe9e35de 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -24,8 +24,6 @@ enum class ConvertTaskType { PLAYER, /// @brief refresh region file version UPGRADE_REGION, - /// @brief rewrite voxels region file to new format - UPGRADE_VOXELS, }; struct ConvertTask { @@ -34,6 +32,7 @@ struct ConvertTask { /// @brief region coords int x, z; + RegionLayerIndex layer; }; class WorldConverter : public Task { @@ -45,8 +44,8 @@ class WorldConverter : public Task { uint tasksDone = 0; bool upgradeMode; - void upgradeRegion(const fs::path& file, int x, int z) const; - void upgradeVoxels(const fs::path& file, int x, int z) const; + void upgradeRegion( + const fs::path& file, int x, int z, RegionLayerIndex layer) const; void convertPlayer(const fs::path& file) const; void convertVoxels(const fs::path& file, int x, int z) const; void convertInventories(const fs::path& file, int x, int z) const; diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index f3e1fd88..500c21a0 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -290,6 +290,10 @@ const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const { return layers[layerid].folder; } +fs::path WorldRegions::getRegionFilePath(RegionLayerIndex layerid, int x, int z) const { + return layers[layerid].getRegionFilePath(x, z); +} + void WorldRegions::writeAll() { for (auto& layer : layers) { fs::create_directories(layer.folder); diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index e4060002..fecd30fa 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -147,6 +147,8 @@ struct RegionsLayer { WorldRegion* getRegion(int x, int z); WorldRegion* getOrCreateRegion(int x, int z); + fs::path getRegionFilePath(int x, int z) const; + /// @brief Get chunk data. Read from file if not loaded yet. /// @param x chunk x coord /// @param z chunk z coord @@ -237,6 +239,8 @@ public: /// @return directory path const fs::path& getRegionsFolder(RegionLayerIndex layerid) const; + fs::path getRegionFilePath(RegionLayerIndex layerid, int x, int z) const; + /// @brief Write all region layers void writeAll(); diff --git a/src/files/compatibility.cpp b/src/files/compatibility.cpp new file mode 100644 index 00000000..0d9d6c08 --- /dev/null +++ b/src/files/compatibility.cpp @@ -0,0 +1,109 @@ +#include "compatibility.hpp" + +#include + +#include "constants.hpp" +#include "voxels/voxel.hpp" +#include "coders/compression.hpp" +#include "coders/byte_utils.hpp" +#include "lighting/Lightmap.hpp" +#include "util/data_io.hpp" + +static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4; +static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4; + +static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t size) { + auto data = compression::decompress( + buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8); + + util::Buffer dstBuffer(VOXELS_DATA_SIZE_V2); + auto dst = reinterpret_cast(dstBuffer.data()); + + for (size_t i = 0; i < CHUNK_VOL; i++) { + ubyte bid1 = data[i]; + ubyte bid2 = data[CHUNK_VOL + i]; + + ubyte bst1 = data[CHUNK_VOL * 2 + i]; + ubyte bst2 = data[CHUNK_VOL * 3 + i]; + + dst[i] = + (static_cast(bid1) << 8) | static_cast(bid2); + dst[CHUNK_VOL + i] = ( + (static_cast(bst1) << 8) | + static_cast(bst2) + ); + } + size_t outLen; + auto compressed = compression::compress( + data.get(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16); + return util::Buffer(std::move(compressed), outLen); +} + +util::Buffer compatibility::convertRegion2to3( + const util::Buffer& src, RegionLayerIndex layer +) { + const size_t REGION_CHUNKS = 1024; + const size_t HEADER_SIZE = 10; + const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t); + const ubyte COMPRESS_NONE = 0; + const ubyte COMPRESS_EXTRLE8 = 1; + const ubyte COMPRESS_EXTRLE16 = 2; + + const ubyte* const ptr = src.data(); + + ByteBuilder builder; + builder.putCStr(".VOXREG"); + builder.put(3); + switch (layer) { + case REGION_LAYER_VOXELS: builder.put(COMPRESS_EXTRLE16); break; + case REGION_LAYER_LIGHTS: builder.put(COMPRESS_EXTRLE8); break; + default: builder.put(COMPRESS_NONE); break; + } + + uint32_t offsets[REGION_CHUNKS] {}; + size_t chunkIndex = 0; + + auto tablePtr = reinterpret_cast( + ptr + src.size() - OFFSET_TABLE_SIZE + ); + + for (size_t i = 0; i < REGION_CHUNKS; i++) { + uint32_t srcOffset = dataio::be2h(tablePtr[i]); + if (srcOffset == 0) { + continue; + } + uint32_t size = *reinterpret_cast(ptr + srcOffset); + size = dataio::be2h(size); + + const ubyte* data = ptr + srcOffset + sizeof(uint32_t); + offsets[i] = builder.size(); + + switch (layer) { + case REGION_LAYER_VOXELS: { + auto dstdata = convert_voxels_1to2(data, size); + builder.putInt32(dstdata.size()); + builder.putInt32(VOXELS_DATA_SIZE_V2); + builder.put(dstdata.data(), dstdata.size()); + break; + } + case REGION_LAYER_LIGHTS: + builder.putInt32(size); + builder.putInt32(LIGHTMAP_DATA_LEN); + builder.put(data, size); + break; + case REGION_LAYER_ENTITIES: + case REGION_LAYER_INVENTORIES: { + builder.putInt32(size); + builder.putInt32(size); + builder.put(data, size); + break; + case REGION_LAYERS_COUNT: + throw std::invalid_argument("invalid enum"); + } + } + } + for (size_t i = 0; i < REGION_CHUNKS; i++) { + builder.putInt32(offsets[i]); + } + return util::Buffer(builder.build().data(), builder.size()); +} diff --git a/src/files/compatibility.hpp b/src/files/compatibility.hpp new file mode 100644 index 00000000..eef5b218 --- /dev/null +++ b/src/files/compatibility.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "typedefs.hpp" +#include "util/Buffer.hpp" +#include "files/world_regions_fwd.hpp" + +namespace compatibility { + /// @brief Convert region file from version 2 to 3 + /// @see /doc/specs/region_file_spec.md + /// @param src region file source content + /// @return new region file content + util::Buffer convertRegion2to3( + const util::Buffer& src, RegionLayerIndex layer); +} diff --git a/src/files/files.cpp b/src/files/files.cpp index e0e52297..8d2a46b4 100644 --- a/src/files/files.cpp +++ b/src/files/files.cpp @@ -66,6 +66,12 @@ bool files::read(const fs::path& filename, char* data, size_t size) { return true; } +util::Buffer files::read_bytes_buffer(const fs::path& path) { + size_t size; + auto bytes = files::read_bytes(path, size); + return util::Buffer(std::move(bytes), size); +} + std::unique_ptr files::read_bytes( const fs::path& filename, size_t& length ) { diff --git a/src/files/files.hpp b/src/files/files.hpp index abcbdbb4..324b1d67 100644 --- a/src/files/files.hpp +++ b/src/files/files.hpp @@ -7,6 +7,7 @@ #include #include "typedefs.hpp" +#include "util/Buffer.hpp" namespace fs = std::filesystem; @@ -59,6 +60,7 @@ namespace files { ); bool read(const fs::path&, char* data, size_t size); + util::Buffer read_bytes_buffer(const fs::path&); std::unique_ptr read_bytes(const fs::path&, size_t& length); std::vector read_bytes(const fs::path&); std::string read_string(const fs::path& filename); diff --git a/test/files/compatibility.cpp b/test/files/compatibility.cpp new file mode 100644 index 00000000..34237e21 --- /dev/null +++ b/test/files/compatibility.cpp @@ -0,0 +1,15 @@ +#include +#include + +#include "files/files.hpp" +#include "files/compatibility.hpp" + +TEST(compatibility, convert) { + auto infile = std::filesystem::u8path( + "voxels_0_1.bin"); + auto outfile = std::filesystem::u8path( + "output_0_1.bin"); + auto input = files::read_bytes_buffer(infile); + auto output = compatibility::convertRegion2to3(input, REGION_LAYER_VOXELS); + files::write_bytes(outfile, output.data(), output.size()); +} From cbb0ecd1f6289f3ae3d65fd91b1a946002745e92 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Wed, 4 Sep 2024 23:41:53 +0300 Subject: [PATCH 32/61] remove compatibility test (requires input file) --- test/files/compatibility.cpp | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 test/files/compatibility.cpp diff --git a/test/files/compatibility.cpp b/test/files/compatibility.cpp deleted file mode 100644 index 34237e21..00000000 --- a/test/files/compatibility.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include - -#include "files/files.hpp" -#include "files/compatibility.hpp" - -TEST(compatibility, convert) { - auto infile = std::filesystem::u8path( - "voxels_0_1.bin"); - auto outfile = std::filesystem::u8path( - "output_0_1.bin"); - auto input = files::read_bytes_buffer(infile); - auto output = compatibility::convertRegion2to3(input, REGION_LAYER_VOXELS); - files::write_bytes(outfile, output.data(), output.size()); -} From c18eddb63c832b8691983a41e9ad9b1b07f8b741 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 5 Sep 2024 11:29:07 +0300 Subject: [PATCH 33/61] add Chunk.encodeV2, decodeV2 --- src/files/WorldConverter.cpp | 2 +- src/files/compatibility.cpp | 2 +- src/files/compatibility.hpp | 2 +- src/voxels/Chunk.cpp | 33 +++++++++++++++++++++++++++++++++ src/voxels/Chunk.hpp | 9 +++++++++ 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index 4748172d..5d24889d 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -161,7 +161,7 @@ void WorldConverter::upgradeRegion( ) const { auto path = wfile->getRegions().getRegionFilePath(layer, x, z); auto bytes = files::read_bytes_buffer(path); - auto buffer = compatibility::convertRegion2to3(bytes, layer); + auto buffer = compatibility::convert_region_2to3(bytes, layer); files::write_bytes(path, buffer.data(), buffer.size()); } diff --git a/src/files/compatibility.cpp b/src/files/compatibility.cpp index 0d9d6c08..161527b1 100644 --- a/src/files/compatibility.cpp +++ b/src/files/compatibility.cpp @@ -39,7 +39,7 @@ static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t siz return util::Buffer(std::move(compressed), outLen); } -util::Buffer compatibility::convertRegion2to3( +util::Buffer compatibility::convert_region_2to3( const util::Buffer& src, RegionLayerIndex layer ) { const size_t REGION_CHUNKS = 1024; diff --git a/src/files/compatibility.hpp b/src/files/compatibility.hpp index eef5b218..cf78b5bc 100644 --- a/src/files/compatibility.hpp +++ b/src/files/compatibility.hpp @@ -9,6 +9,6 @@ namespace compatibility { /// @see /doc/specs/region_file_spec.md /// @param src region file source content /// @return new region file content - util::Buffer convertRegion2to3( + util::Buffer convert_region_2to3( const util::Buffer& src, RegionLayerIndex layer); } diff --git a/src/voxels/Chunk.cpp b/src/voxels/Chunk.cpp index c77532da..5dfb9fe3 100644 --- a/src/voxels/Chunk.cpp +++ b/src/voxels/Chunk.cpp @@ -5,6 +5,7 @@ #include "content/ContentReport.hpp" #include "items/Inventory.hpp" #include "lighting/Lightmap.hpp" +#include "util/data_io.hpp" #include "voxel.hpp" Chunk::Chunk(int xpos, int zpos) : x(xpos), z(zpos) { @@ -103,6 +104,27 @@ std::unique_ptr Chunk::encode() const { return buffer; } +/** + Current chunk format: + - byte-order: little-endian + + ```cpp + uint16_t voxel_id[CHUNK_VOL]; + uint16_t voxel_states[CHUNK_VOL]; + ``` + + Total size: (CHUNK_VOL * 4) bytes +*/ +std::unique_ptr Chunk::encodeV2() const { + auto buffer = std::make_unique(CHUNK_DATA_LEN); + auto dst = reinterpret_cast(buffer.get()); + for (uint i = 0; i < CHUNK_VOL; i++) { + dst[i] = dataio::h2le(voxels[i].id); + dst[CHUNK_VOL + i] = dataio::h2le(blockstate2int(voxels[i].state)); + } + return buffer; +} + bool Chunk::decode(const ubyte* data) { for (uint i = 0; i < CHUNK_VOL; i++) { voxel& vox = voxels[i]; @@ -123,6 +145,17 @@ bool Chunk::decode(const ubyte* data) { return true; } +bool Chunk::decodeV2(const ubyte* data) { + auto src = reinterpret_cast(data); + for (uint i = 0; i < CHUNK_VOL; i++) { + voxel& vox = voxels[i]; + + vox.id = dataio::le2h(src[i]); + vox.state = int2blockstate(dataio::le2h(src[CHUNK_VOL + i])); + } + return true; +} + void Chunk::convert(ubyte* data, const ContentReport* report) { for (uint i = 0; i < CHUNK_VOL; i++) { // see encode method to understand what the hell is going on here diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index d9a25186..bd8bf965 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -66,10 +66,19 @@ public: flags.unsaved = true; } + /// @brief Encode chunk to bytes array of size CHUNK_DATA_LEN + /// @see /doc/specs/outdated/region_voxels_chunk_spec_v1.md std::unique_ptr encode() const; + /// @brief Encode chunk to bytes array of size CHUNK_DATA_LEN + /// @see /doc/specs/region_voxels_chunk_spec.md + std::unique_ptr encodeV2() const; + /// @return true if all is fine bool decode(const ubyte* data); + /// @return true if all is fine + bool decodeV2(const ubyte* data); + static void convert(ubyte* data, const ContentReport* report); }; From 4a619fc18eb020c18774e73f505afa3f77d27cc7 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Thu, 5 Sep 2024 11:34:01 +0300 Subject: [PATCH 34/61] add Chunk.convertV2 --- src/voxels/Chunk.cpp | 10 +++++++++- src/voxels/Chunk.hpp | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/voxels/Chunk.cpp b/src/voxels/Chunk.cpp index 5dfb9fe3..80bfcd73 100644 --- a/src/voxels/Chunk.cpp +++ b/src/voxels/Chunk.cpp @@ -158,7 +158,6 @@ bool Chunk::decodeV2(const ubyte* data) { void Chunk::convert(ubyte* data, const ContentReport* report) { for (uint i = 0; i < CHUNK_VOL; i++) { - // see encode method to understand what the hell is going on here blockid_t id = ((static_cast(data[i]) << 8) | static_cast(data[CHUNK_VOL + i])); @@ -167,3 +166,12 @@ void Chunk::convert(ubyte* data, const ContentReport* report) { data[CHUNK_VOL + i] = replacement & 0xFF; } } + +void Chunk::convertV2(ubyte* data, const ContentReport* report) { + auto buffer = reinterpret_cast(data); + for (uint i = 0; i < CHUNK_VOL; i++) { + blockid_t id = dataio::le2h(buffer[i]); + blockid_t replacement = report->blocks.getId(id); + buffer[i] = dataio::h2le(replacement); + } +} diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index bd8bf965..382c69f3 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -81,4 +81,5 @@ public: bool decodeV2(const ubyte* data); static void convert(ubyte* data, const ContentReport* report); + static void convertV2(ubyte* data, const ContentReport* report); }; From 69b90f53c386a0a304a4ad6202a843d6e36477fb Mon Sep 17 00:00:00 2001 From: MihailRis Date: Fri, 6 Sep 2024 12:08:10 +0300 Subject: [PATCH 35/61] fix extRLE16 encoder --- src/coders/rle.cpp | 2 +- test/coders/rle.cpp | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/coders/rle.cpp b/src/coders/rle.cpp index f049492e..e99006e5 100644 --- a/src/coders/rle.cpp +++ b/src/coders/rle.cpp @@ -158,7 +158,7 @@ size_t extrle::encode16(const ubyte* src8, size_t srclen, ubyte* dst) { uint16_t c = src[0]; for (size_t i = 1; i < srclen/2; i++) { uint16_t cnext = src[i]; - if (cnext != c || counter == max_sequence) { + if (cnext != c || counter == max_sequence16) { if (counter >= 0x40) { dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F); dst[offset++] = counter >> 6; diff --git a/test/coders/rle.cpp b/test/coders/rle.cpp index 3790fdb1..e393f8dd 100644 --- a/test/coders/rle.cpp +++ b/test/coders/rle.cpp @@ -5,14 +5,15 @@ static void test_encode_decode( size_t(*encodefunc)(const ubyte*, size_t, ubyte*), - size_t(*decodefunc)(const ubyte*, size_t, ubyte*) + size_t(*decodefunc)(const ubyte*, size_t, ubyte*), + int dencity ) { const size_t initial_size = 50'000; uint8_t initial[initial_size]; uint8_t next = rand(); for (size_t i = 0; i < initial_size; i++) { initial[i] = next; - if (rand() % 13 == 0) { + if (rand() % dencity == 0) { next = rand(); } } @@ -29,17 +30,21 @@ static void test_encode_decode( } TEST(RLE, EncodeDecode) { - test_encode_decode(rle::encode, rle::decode); + test_encode_decode(rle::encode, rle::decode, 13); + test_encode_decode(rle::encode, rle::decode, 90123); } TEST(RLE16, EncodeDecode) { - test_encode_decode(rle::encode16, rle::decode16); + test_encode_decode(rle::encode16, rle::decode16, 13); + test_encode_decode(rle::encode16, rle::decode16, 90123); } TEST(ExtRLE, EncodeDecode) { - test_encode_decode(extrle::encode, extrle::decode); + test_encode_decode(extrle::encode, extrle::decode, 13); + test_encode_decode(extrle::encode, extrle::decode, 90123); } TEST(ExtRLE16, EncodeDecode) { - test_encode_decode(extrle::encode16, extrle::decode16); + test_encode_decode(extrle::encode16, extrle::decode16, 13); + test_encode_decode(extrle::encode16, extrle::decode16, 90123); } From e30c1b3c03a3357ac31bf0e6fa63282f064f8386 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Fri, 6 Sep 2024 12:25:52 +0300 Subject: [PATCH 36/61] upgrade regions format version to 3 --- res/texts/en_US.txt | 1 + res/texts/ru_RU.txt | 1 + src/coders/compression.cpp | 7 ++- src/constants.hpp | 2 +- src/content/ContentReport.cpp | 14 +++++- src/files/RegionsLayer.cpp | 82 +++++++++++++++++++--------------- src/files/WorldConverter.cpp | 11 +++-- src/files/WorldFiles.cpp | 11 +++++ src/files/WorldFiles.hpp | 2 + src/files/WorldRegions.cpp | 54 +++++++++++----------- src/files/WorldRegions.hpp | 22 ++++----- src/files/compatibility.cpp | 7 ++- src/logic/EngineController.cpp | 3 +- src/voxels/Chunk.cpp | 60 +------------------------ src/voxels/Chunk.hpp | 10 +---- 15 files changed, 138 insertions(+), 149 deletions(-) diff --git a/res/texts/en_US.txt b/res/texts/en_US.txt index f36f12a7..9cf71339 100644 --- a/res/texts/en_US.txt +++ b/res/texts/en_US.txt @@ -1,6 +1,7 @@ # Menu menu.missing-content=Missing Content! world.convert-request=Content indices have changed! Convert world files? +world.upgrade-request=World format is outdated! Convert world files? pack.remove-confirm=Do you want to erase all pack(s) content from the world forever? error.pack-not-found=Could not to find pack error.dependency-not-found=Dependency pack is not found diff --git a/res/texts/ru_RU.txt b/res/texts/ru_RU.txt index f8abdd9b..20021b53 100644 --- a/res/texts/ru_RU.txt +++ b/res/texts/ru_RU.txt @@ -46,6 +46,7 @@ world.generators.default=Обычный world.generators.flat=Плоский world.Create World=Создать Мир world.convert-request=Есть изменения в индексах! Конвертировать мир? +world.upgrade-request=Формат мира устарел! Конвертировать мир? world.delete-confirm=Удалить мир безвозвратно? # Настройки diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp index 74f68528..9f7b551e 100644 --- a/src/coders/compression.cpp +++ b/src/coders/compression.cpp @@ -82,7 +82,12 @@ std::unique_ptr compression::decompress( } case Method::EXTRLE16: { auto decompressed = std::make_unique(dstlen); - extrle::decode16(src, srclen, decompressed.get()); + size_t decoded = extrle::decode16(src, srclen, decompressed.get()); + if (decoded != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(decoded)); + } return decompressed; } case Method::GZIP: { diff --git a/src/constants.hpp b/src/constants.hpp index b1ae11d4..da4acfc9 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -17,7 +17,7 @@ inline constexpr bool ENGINE_DEBUG_BUILD = true; inline const std::string ENGINE_VERSION_STRING = "0.23"; /// @brief world regions format version -inline constexpr uint REGION_FORMAT_VERSION = 2; +inline constexpr uint REGION_FORMAT_VERSION = 3; /// @brief max simultaneously open world region files inline constexpr uint MAX_OPEN_REGION_FILES = 32; diff --git a/src/content/ContentReport.cpp b/src/content/ContentReport.cpp index 1e524309..a8fc806d 100644 --- a/src/content/ContentReport.cpp +++ b/src/content/ContentReport.cpp @@ -55,7 +55,9 @@ std::shared_ptr ContentReport::create( report->items.setup(itemlist.get(), content->items); report->buildIssues(); - if (report->hasContentReorder() || report->hasMissingContent()) { + if (report->isUpgradeRequired() || + report->hasContentReorder() || + report->hasMissingContent()) { return report; } else { return nullptr; @@ -79,6 +81,16 @@ static void build_issues( void ContentReport::buildIssues() { build_issues(issues, blocks); build_issues(issues, items); + + if (regionsVersion < REGION_FORMAT_VERSION) { + for (int layer = REGION_LAYER_VOXELS; + layer < REGION_LAYERS_COUNT; + layer++) { + ContentIssue issue {ContentIssueType::REGION_FORMAT_UPDATE}; + issue.regionLayer = static_cast(layer); + issues.push_back(issue); + } + } } const std::vector& ContentReport::getIssues() const { diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp index 3f0101f4..027d6bf3 100644 --- a/src/files/RegionsLayer.cpp +++ b/src/files/RegionsLayer.cpp @@ -13,14 +13,14 @@ static fs::path get_region_filename(int x, int z) { /// @brief Read missing chunks data (null pointers) from region file static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) { auto* chunks = region->getChunks(); - uint32_t* sizes = region->getSizes(); + auto sizes = region->getSizes(); for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; if (chunks[i] == nullptr) { - chunks[i] = - RegionsLayer::readChunkData(chunk_x, chunk_z, sizes[i], file); + chunks[i] = RegionsLayer::readChunkData( + chunk_x, chunk_z, sizes[i][0], sizes[i][1], file); } } } @@ -44,23 +44,26 @@ regfile::regfile(fs::path filename) : file(std::move(filename)) { } } -std::unique_ptr regfile::read(int index, uint32_t& length) { +std::unique_ptr regfile::read(int index, uint32_t& size, uint32_t& srcSize) { size_t file_size = file.length(); size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; - uint32_t offset; + uint32_t buff32; file.seekg(table_offset + index * 4); - file.read(reinterpret_cast(&offset), 4); - offset = dataio::read_int32_big(reinterpret_cast(&offset), 0); + file.read(reinterpret_cast(&buff32), 4); + uint32_t offset = dataio::le2h(buff32); if (offset == 0) { return nullptr; } file.seekg(offset); - file.read(reinterpret_cast(&offset), 4); - length = dataio::read_int32_big(reinterpret_cast(&offset), 0); - auto data = std::make_unique(length); - file.read(reinterpret_cast(data.get()), length); + file.read(reinterpret_cast(&buff32), 4); + size = dataio::le2h(buff32); + file.read(reinterpret_cast(&buff32), 4); + srcSize = dataio::le2h(buff32); + + auto data = std::make_unique(size); + file.read(reinterpret_cast(data.get()), size); return data; } @@ -150,7 +153,7 @@ WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) { return region; } -ubyte* RegionsLayer::getData(int x, int z, uint32_t& size) { +ubyte* RegionsLayer::getData(int x, int z, uint32_t& size, uint32_t& srcSize) { int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); @@ -159,15 +162,17 @@ ubyte* RegionsLayer::getData(int x, int z, uint32_t& size) { if (data == nullptr) { auto regfile = getRegFile({regionX, regionZ}); if (regfile != nullptr) { - auto dataptr = readChunkData(x, z, size, regfile.get()); + auto dataptr = readChunkData(x, z, size, srcSize, regfile.get()); if (dataptr) { data = dataptr.get(); - region->put(localX, localZ, std::move(dataptr), size); + region->put(localX, localZ, std::move(dataptr), size, srcSize); } } } if (data != nullptr) { - size = region->getChunkDataSize(localX, localZ); + auto sizevec = region->getChunkDataSize(localX, localZ); + size = sizevec[0]; + srcSize = sizevec[1]; return data; } return nullptr; @@ -187,47 +192,50 @@ void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) { char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; header[8] = REGION_FORMAT_VERSION; - header[9] = 0; // flags + header[9] = static_cast(compression); // FIXME std::ofstream file(filename, std::ios::out | std::ios::binary); file.write(header, REGION_HEADER_SIZE); size_t offset = REGION_HEADER_SIZE; - char intbuf[4] {}; + uint32_t intbuf; uint offsets[REGION_CHUNKS_COUNT] {}; - auto* region = entry->getChunks(); - uint32_t* sizes = entry->getSizes(); + auto region = entry->getChunks(); + auto sizes = entry->getSizes(); for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { ubyte* chunk = region[i].get(); if (chunk == nullptr) { - offsets[i] = 0; - } else { - offsets[i] = offset; - - size_t compressedSize = sizes[i]; - dataio::write_int32_big( - compressedSize, reinterpret_cast(intbuf), 0 - ); - offset += 4 + compressedSize; - - file.write(intbuf, 4); - file.write(reinterpret_cast(chunk), compressedSize); + continue; } + offsets[i] = offset; + + auto sizevec = sizes[i]; + uint32_t compressedSize = sizevec[0]; + uint32_t srcSize = sizevec[1]; + + intbuf = dataio::h2le(compressedSize); + file.write(reinterpret_cast(&intbuf), 4); + offset += 4; + + intbuf = dataio::h2le(srcSize); + file.write(reinterpret_cast(&intbuf), 4); + offset += 4; + + file.write(reinterpret_cast(chunk), compressedSize); + offset += compressedSize; } for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - dataio::write_int32_big( - offsets[i], reinterpret_cast(intbuf), 0 - ); - file.write(intbuf, 4); + intbuf = dataio::h2le(offsets[i]); + file.write(reinterpret_cast(&intbuf), 4); } } std::unique_ptr RegionsLayer::readChunkData( - int x, int z, uint32_t& length, regfile* rfile + int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile ) { int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); int chunkIndex = localZ * REGION_SIZE + localX; - return rfile->read(chunkIndex, length); + return rfile->read(chunkIndex, size, srcSize); } diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index 5d24889d..396f1f16 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -167,7 +167,7 @@ void WorldConverter::upgradeRegion( void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const { logger.info() << "converting voxels region " << x << "_" << z; - wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, CHUNK_DATA_LEN, + wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, [=](std::unique_ptr data, uint32_t*) { Chunk::convert(data.get(), report.get()); return data; @@ -238,8 +238,13 @@ bool WorldConverter::isActive() const { } void WorldConverter::write() { - logger.info() << "writing world"; - wfile->write(nullptr, upgradeMode ? nullptr : content); + if (upgradeMode) { + logger.info() << "refreshing version"; + wfile->patchIndicesVersion("region-version", REGION_FORMAT_VERSION); + } else { + logger.info() << "writing world"; + wfile->write(nullptr, content); + } } void WorldConverter::waitForEnd() { diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 54e96759..8fb1913d 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -172,6 +172,17 @@ bool WorldFiles::readResourcesData(const Content* content) { return true; } +void WorldFiles::patchIndicesVersion(const std::string& field, uint version) { + fs::path file = getIndicesFile(); + if (!fs::is_regular_file(file)) { + logger.error() << file.filename().u8string() << " does not exists"; + return; + } + auto root = files::read_json(file); + root->put(field, version); + files::write_json(file, root.get(), true); +} + static void erase_pack_indices(dynamic::Map* root, const std::string& id) { auto prefix = id + ":"; auto blocks = root->list("blocks"); diff --git a/src/files/WorldFiles.hpp b/src/files/WorldFiles.hpp index cb9c4b45..32b64060 100644 --- a/src/files/WorldFiles.hpp +++ b/src/files/WorldFiles.hpp @@ -51,6 +51,8 @@ public: std::optional readWorldInfo(); bool readResourcesData(const Content* content); + void patchIndicesVersion(const std::string& field, uint version); + /// @brief Write all unsaved data to world files /// @param world target world /// @param content world content diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 500c21a0..d0fb57d3 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -14,7 +14,7 @@ WorldRegion::WorldRegion() : chunksData( std::make_unique[]>(REGION_CHUNKS_COUNT) ), - sizes(std::make_unique(REGION_CHUNKS_COUNT)) { + sizes(std::make_unique(REGION_CHUNKS_COUNT)) { } WorldRegion::~WorldRegion() = default; @@ -30,23 +30,23 @@ std::unique_ptr* WorldRegion::getChunks() const { return chunksData.get(); } -uint32_t* WorldRegion::getSizes() const { +glm::u32vec2* WorldRegion::getSizes() const { return sizes.get(); } void WorldRegion::put( - uint x, uint z, std::unique_ptr data, uint32_t size + uint x, uint z, std::unique_ptr data, uint32_t size, uint32_t srcSize ) { size_t chunk_index = z * REGION_SIZE + x; chunksData[chunk_index] = std::move(data); - sizes[chunk_index] = size; + sizes[chunk_index] = glm::u32vec2(size, srcSize); } ubyte* WorldRegion::getChunkData(uint x, uint z) { return chunksData[z * REGION_SIZE + x].get(); } -uint WorldRegion::getChunkDataSize(uint x, uint z) { +glm::u32vec2 WorldRegion::getChunkDataSize(uint x, uint z) { return sizes[z * REGION_SIZE + x]; } @@ -56,7 +56,7 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { } auto& voxels = layers[REGION_LAYER_VOXELS]; voxels.folder = directory / fs::path("regions"); - voxels.compression = compression::Method::EXTRLE8; + voxels.compression = compression::Method::EXTRLE16; auto& lights = layers[REGION_LAYER_LIGHTS]; lights.folder = directory / fs::path("lights"); @@ -85,8 +85,9 @@ void WorldRegions::put( int z, RegionLayerIndex layerid, std::unique_ptr data, - size_t size + size_t srcSize ) { + size_t size = srcSize; auto& layer = layers[layerid]; if (layer.compression != compression::Method::NONE) { data = compression::compress( @@ -97,7 +98,7 @@ void WorldRegions::put( WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); - region->put(localX, localZ, std::move(data), size); + region->put(localX, localZ, std::move(data), size, srcSize); } static std::unique_ptr write_inventories( @@ -188,30 +189,35 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { std::unique_ptr WorldRegions::getVoxels(int x, int z) { uint32_t size; + uint32_t srcSize; auto& layer = layers[REGION_LAYER_VOXELS]; - auto* data = layer.getData(x, z, size); + auto* data = layer.getData(x, z, size, srcSize); if (data == nullptr) { return nullptr; } - return compression::decompress(data, size, CHUNK_DATA_LEN, layer.compression); + assert(srcSize == CHUNK_DATA_LEN); + return compression::decompress(data, size, srcSize, layer.compression); } std::unique_ptr WorldRegions::getLights(int x, int z) { uint32_t size; + uint32_t srcSize; auto& layer = layers[REGION_LAYER_LIGHTS]; - auto* bytes = layer.getData(x, z, size); + auto* bytes = layer.getData(x, z, size, srcSize); if (bytes == nullptr) { return nullptr; } auto data = compression::decompress( - bytes, size, LIGHTMAP_DATA_LEN, layer.compression + bytes, size, srcSize, layer.compression ); + assert(srcSize == LIGHTMAP_DATA_LEN); return Lightmap::decode(data.get()); } chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { uint32_t bytesSize; - auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize); + uint32_t srcSize; + auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize, srcSize); if (bytes == nullptr) { return {}; } @@ -221,7 +227,7 @@ chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { void WorldRegions::processInventories( int x, int z, const inventoryproc& func ) { - processRegion(x, z, REGION_LAYER_INVENTORIES, 0, + processRegion(x, z, REGION_LAYER_INVENTORIES, [=](std::unique_ptr data, uint32_t* size) { auto inventories = load_inventories(data.get(), *size); for (const auto& [_, inventory] : inventories) { @@ -236,7 +242,8 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { return nullptr; } uint32_t bytesSize; - const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize); + uint32_t srcSize; + const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize, srcSize); if (data == nullptr) { return nullptr; } @@ -248,7 +255,7 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { } void WorldRegions::processRegion( - int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func + int x, int z, RegionLayerIndex layerid, const regionproc& func ) { auto& layer = layers[layerid]; if (layer.getRegion(x, z)) { @@ -263,24 +270,21 @@ void WorldRegions::processRegion( int gx = cx + x * REGION_SIZE; int gz = cz + z * REGION_SIZE; uint32_t length; + uint32_t srcSize; auto data = - RegionsLayer::readChunkData(gx, gz, length, regfile.get()); + RegionsLayer::readChunkData(gx, gz, length, srcSize, regfile.get()); if (data == nullptr) { continue; } - uint32_t totalLength = dataLen; if (layer.compression != compression::Method::NONE) { - if (dataLen == 0) { - throw std::invalid_argument("invalid data length"); - } data = compression::decompress( - data.get(), length, dataLen, layer.compression + data.get(), length, srcSize, layer.compression ); } else { - totalLength = length; + srcSize = length; } - if (auto writeData = func(std::move(data), &totalLength)) { - put(gx, gz, layerid, std::move(writeData), totalLength); + if (auto writeData = func(std::move(data), &srcSize)) { + put(gx, gz, layerid, std::move(writeData), srcSize); } } } diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index fecd30fa..511107bf 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -38,21 +38,21 @@ public: class WorldRegion { std::unique_ptr[]> chunksData; - std::unique_ptr sizes; + std::unique_ptr sizes; bool unsaved = false; public: WorldRegion(); ~WorldRegion(); - void put(uint x, uint z, std::unique_ptr data, uint32_t size); + void put(uint x, uint z, std::unique_ptr data, uint32_t size, uint32_t srcSize); ubyte* getChunkData(uint x, uint z); - uint getChunkDataSize(uint x, uint z); + glm::u32vec2 getChunkDataSize(uint x, uint z); void setUnsaved(bool unsaved); bool isUnsaved() const; std::unique_ptr* getChunks() const; - uint32_t* getSizes() const; + glm::u32vec2* getSizes() const; }; struct regfile { @@ -63,7 +63,7 @@ struct regfile { regfile(fs::path filename); regfile(const regfile&) = delete; - std::unique_ptr read(int index, uint32_t& length); + std::unique_ptr read(int index, uint32_t& size, uint32_t& srcSize); }; using regionsmap = std::unordered_map>; @@ -152,9 +152,10 @@ struct RegionsLayer { /// @brief Get chunk data. Read from file if not loaded yet. /// @param x chunk x coord /// @param z chunk z coord - /// @param size [out] chunk data length + /// @param size [out] compressed chunk data length + /// @param size [out] source chunk data length /// @return nullptr if no saved chunk data found - [[nodiscard]] ubyte* getData(int x, int z, uint32_t& size); + [[nodiscard]] ubyte* getData(int x, int z, uint32_t& size, uint32_t& srcSize); /// @brief Write or rewrite region file /// @param x region X @@ -167,11 +168,12 @@ struct RegionsLayer { /// @brief Read chunk data from region file /// @param x chunk x coord /// @param z chunk z coord - /// @param length [out] chunk data length + /// @param size [out] compressed chunk data length + /// @param srcSize [out] source chunk data length /// @param rfile region file /// @return nullptr if chunk is not present in region file [[nodiscard]] static std::unique_ptr readChunkData( - int x, int z, uint32_t& length, regfile* rfile + int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile ); }; @@ -229,7 +231,7 @@ public: /// @param layerid regions layer index /// @param func processing callback void processRegion( - int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func); + int x, int z, RegionLayerIndex layerid, const regionproc& func); void processInventories( int x, int z, const inventoryproc& func); diff --git a/src/files/compatibility.cpp b/src/files/compatibility.cpp index 161527b1..8cdd437d 100644 --- a/src/files/compatibility.cpp +++ b/src/files/compatibility.cpp @@ -12,6 +12,7 @@ static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4; static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4; +#include static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t size) { auto data = compression::decompress( buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8); @@ -30,18 +31,20 @@ static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t siz (static_cast(bid1) << 8) | static_cast(bid2); dst[CHUNK_VOL + i] = ( (static_cast(bst1) << 8) | - static_cast(bst2) + static_cast(bst2) ); } size_t outLen; auto compressed = compression::compress( - data.get(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16); + dstBuffer.data(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16); return util::Buffer(std::move(compressed), outLen); } +#include "util/timeutil.hpp" util::Buffer compatibility::convert_region_2to3( const util::Buffer& src, RegionLayerIndex layer ) { + timeutil::ScopeLogTimer log(555); const size_t REGION_CHUNKS = 1024; const size_t HEADER_SIZE = 10; const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t); diff --git a/src/logic/EngineController.cpp b/src/logic/EngineController.cpp index 8cb53f28..bbdb06eb 100644 --- a/src/logic/EngineController.cpp +++ b/src/logic/EngineController.cpp @@ -73,7 +73,8 @@ void show_convert_request( ) { guiutil::confirm( engine->getGUI(), - langs::get(L"world.convert-request"), + langs::get(report->isUpgradeRequired() ? + L"world.upgrade-request" : L"world.convert-request"), [=]() { auto converter = create_converter(engine, worldFiles, content, report, postRunnable); diff --git a/src/voxels/Chunk.cpp b/src/voxels/Chunk.cpp index 80bfcd73..238e2975 100644 --- a/src/voxels/Chunk.cpp +++ b/src/voxels/Chunk.cpp @@ -77,33 +77,6 @@ std::unique_ptr Chunk::clone() const { return other; } -/** - Current chunk format: - - byte-order: big-endian - - [don't panic!] first and second bytes are separated for RLE efficiency - - ```cpp - uint8_t voxel_id_first_byte[CHUNK_VOL]; - uint8_t voxel_id_second_byte[CHUNK_VOL]; - uint8_t voxel_states_first_byte[CHUNK_VOL]; - uint8_t voxel_states_second_byte[CHUNK_VOL]; - ``` - - Total size: (CHUNK_VOL * 4) bytes -*/ -std::unique_ptr Chunk::encode() const { - auto buffer = std::make_unique(CHUNK_DATA_LEN); - for (uint i = 0; i < CHUNK_VOL; i++) { - buffer[i] = voxels[i].id >> 8; - buffer[CHUNK_VOL + i] = voxels[i].id & 0xFF; - - blockstate_t state = blockstate2int(voxels[i].state); - buffer[CHUNK_VOL * 2 + i] = state >> 8; - buffer[CHUNK_VOL * 3 + i] = state & 0xFF; - } - return buffer; -} - /** Current chunk format: - byte-order: little-endian @@ -115,7 +88,7 @@ std::unique_ptr Chunk::encode() const { Total size: (CHUNK_VOL * 4) bytes */ -std::unique_ptr Chunk::encodeV2() const { +std::unique_ptr Chunk::encode() const { auto buffer = std::make_unique(CHUNK_DATA_LEN); auto dst = reinterpret_cast(buffer.get()); for (uint i = 0; i < CHUNK_VOL; i++) { @@ -126,26 +99,6 @@ std::unique_ptr Chunk::encodeV2() const { } bool Chunk::decode(const ubyte* data) { - for (uint i = 0; i < CHUNK_VOL; i++) { - voxel& vox = voxels[i]; - - ubyte bid1 = data[i]; - ubyte bid2 = data[CHUNK_VOL + i]; - - ubyte bst1 = data[CHUNK_VOL * 2 + i]; - ubyte bst2 = data[CHUNK_VOL * 3 + i]; - - vox.id = - (static_cast(bid1) << 8) | static_cast(bid2); - vox.state = int2blockstate( - (static_cast(bst1) << 8) | - static_cast(bst2) - ); - } - return true; -} - -bool Chunk::decodeV2(const ubyte* data) { auto src = reinterpret_cast(data); for (uint i = 0; i < CHUNK_VOL; i++) { voxel& vox = voxels[i]; @@ -157,17 +110,6 @@ bool Chunk::decodeV2(const ubyte* data) { } void Chunk::convert(ubyte* data, const ContentReport* report) { - for (uint i = 0; i < CHUNK_VOL; i++) { - blockid_t id = - ((static_cast(data[i]) << 8) | - static_cast(data[CHUNK_VOL + i])); - blockid_t replacement = report->blocks.getId(id); - data[i] = replacement >> 8; - data[CHUNK_VOL + i] = replacement & 0xFF; - } -} - -void Chunk::convertV2(ubyte* data, const ContentReport* report) { auto buffer = reinterpret_cast(data); for (uint i = 0; i < CHUNK_VOL; i++) { blockid_t id = dataio::le2h(buffer[i]); diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index 382c69f3..79f75c28 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -66,20 +66,12 @@ public: flags.unsaved = true; } - /// @brief Encode chunk to bytes array of size CHUNK_DATA_LEN - /// @see /doc/specs/outdated/region_voxels_chunk_spec_v1.md - std::unique_ptr encode() const; - /// @brief Encode chunk to bytes array of size CHUNK_DATA_LEN /// @see /doc/specs/region_voxels_chunk_spec.md - std::unique_ptr encodeV2() const; + std::unique_ptr encode() const; /// @return true if all is fine bool decode(const ubyte* data); - /// @return true if all is fine - bool decodeV2(const ubyte* data); - static void convert(ubyte* data, const ContentReport* report); - static void convertV2(ubyte* data, const ContentReport* report); }; From f5d2a1f1b7e07efb8f8ffa71a3ebfefe81a00224 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 14 Sep 2024 18:56:37 +0300 Subject: [PATCH 37/61] make StructLayout serializable --- src/data/StructLayout.cpp | 61 +++++++++++++++++++++++++++++++++ src/data/StructLayout.hpp | 51 ++++++++++++++++++++++++--- src/interfaces/Serializable.hpp | 2 +- src/util/SmallHeap.hpp | 11 ++++-- test/data/StructLayout.cpp | 17 +++++++++ 5 files changed, 134 insertions(+), 8 deletions(-) diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 9124b68a..b434b326 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -1,5 +1,6 @@ #include "StructLayout.hpp" +#include #include #include #include @@ -13,6 +14,27 @@ using namespace data; static_assert(sizeof(float) == sizeof(int32_t)); static_assert(sizeof(double) == sizeof(int64_t)); +FieldType data::FieldType_from_string(std::string_view name) { + std::map map { + {"int8", FieldType::I8}, + {"int16", FieldType::I16}, + {"int32", FieldType::I32}, + {"int64", FieldType::I64}, + {"float32", FieldType::F32}, + {"float64", FieldType::F64}, + {"char", FieldType::CHAR}, + }; + return map.at(name); +} + +FieldConvertStrategy data::FieldConvertStrategy_from_string(std::string_view name) { + std::map map { + {"reset", FieldConvertStrategy::RESET}, + {"clamp", FieldConvertStrategy::CLAMP} + }; + return map.at(name); +} + StructLayout StructLayout::create(const std::vector& fields) { std::vector builtFields = fields; std::unordered_map indices; @@ -325,3 +347,42 @@ std::string_view StructLayout::getChars( auto ptr = reinterpret_cast(src + field.offset); return std::string_view(ptr, strnlen(ptr, field.elements)); } + +std::unique_ptr StructLayout::serialize() const { + auto map = std::make_unique(); + for (const auto& [name, index] : indices) { + auto& fieldmap = map->putMap(name); + const auto& field = fields[index]; + fieldmap.put("type", to_string(field.type)); + if (field.elements != 1) { + fieldmap.put("length", field.elements); + } + if (field.convertStrategy != FieldConvertStrategy::RESET) { + fieldmap.put("convert-strategy", to_string(field.convertStrategy)); + } + } + return map; +} + +void StructLayout::deserialize(dynamic::Map* src) { + std::vector fields; + for (auto& [name, value] : src->values) { + if (auto fieldmapptr = std::get_if(&value)) { + const auto& fieldmap = *fieldmapptr; + + auto typeName = fieldmap->get("type"); + FieldType type = FieldType_from_string(typeName); + + int elements = fieldmap->get("length", 1); + + auto convertStrategy = FieldConvertStrategy::RESET; + if (fieldmap->has("convert-strategy")) { + convertStrategy = FieldConvertStrategy_from_string( + fieldmap->get("convert-strategy") + ); + } + fields.push_back(Field {type, name, elements, convertStrategy}); + } + } + *this = create(fields); +} diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index b30bcea6..35942457 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -7,6 +7,7 @@ #include #include "typedefs.hpp" +#include "interfaces/Serializable.hpp" namespace data { enum class FieldType { @@ -14,6 +15,14 @@ namespace data { COUNT }; + inline const char* to_string(FieldType type) { + const char* names[] = { + "int8", "int16", "int32", "int64", "float32", "float64", "char" + }; + return names[static_cast(type)]; + } + FieldType FieldType_from_string(std::string_view name); + /// @brief Sorted by severity enum class FieldIncapatibilityType { NONE = 0, @@ -28,7 +37,7 @@ namespace data { }; inline constexpr int sizeof_type(FieldType type) { - const int sizes[static_cast(FieldType::COUNT)] = { + const int sizes[] = { 1, 2, 4, 8, 4, 8, 1 }; return sizes[static_cast(type)]; @@ -42,10 +51,18 @@ namespace data { /// @brief Strategy will be used if value can't be left the same on conversion enum class FieldConvertStrategy { /// @brief Reset field value - RESET, + RESET = 0, /// @brief Clamp field value if out of type range CLAMP }; + + inline const char* to_string(FieldConvertStrategy strategy) { + const char* names[] = { + "reset", "clamp" + }; + return names[static_cast(strategy)]; + } + FieldConvertStrategy FieldConvertStrategy_from_string(std::string_view name); struct Field { FieldType type; @@ -58,13 +75,25 @@ namespace data { int offset; /// @brief Byte size of the field int size; + + bool operator==(const Field& o) const { + return type == o.type && + name == o.name && + elements == o.elements && + convertStrategy == o.convertStrategy && + offset == o.offset && + size == o.size; + } + bool operator!=(const Field& o) const { + return !operator==(o); + } }; - class StructLayout { + class StructLayout : public Serializable { int totalSize; std::vector fields; std::unordered_map indices; - public: + StructLayout( int totalSize, std::vector fields, @@ -73,6 +102,16 @@ namespace data { fields(std::move(fields)), indices(std::move(indices)) {} + public: + StructLayout() : StructLayout(0, {}, {}) {} + + bool operator==(const StructLayout& o) const { + // if fields are completely equal then structures are equal + return fields == o.fields; + } + bool operator!=(const StructLayout& o) const { + return !operator==(o); + } /// @brief Get field by name. Returns nullptr if field not found. /// @param name field name @@ -83,7 +122,6 @@ namespace data { if (found == indices.end()) { return nullptr; } - return &fields.at(found->second); } @@ -183,5 +221,8 @@ namespace data { [[nodiscard]] static StructLayout create(const std::vector& fields); + + std::unique_ptr serialize() const override; + void deserialize(dynamic::Map* src) override; }; } diff --git a/src/interfaces/Serializable.hpp b/src/interfaces/Serializable.hpp index 89962079..7c15fffa 100644 --- a/src/interfaces/Serializable.hpp +++ b/src/interfaces/Serializable.hpp @@ -7,6 +7,6 @@ class Serializable { public: virtual ~Serializable() { } - virtual std::unique_ptr serialize() const = 0; + virtual std::unique_ptr serialize() const = 0; virtual void deserialize(dynamic::Map* src) = 0; }; diff --git a/src/util/SmallHeap.hpp b/src/util/SmallHeap.hpp index aea3f51d..0279f465 100644 --- a/src/util/SmallHeap.hpp +++ b/src/util/SmallHeap.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace util { @@ -62,9 +63,15 @@ namespace util { /// @param size entry size /// @return temporary entry pointer /// @attention pointer becomes invalid after allocate(...) or free(...) - uint8_t* allocate(Tindex index, Tsize size) { + uint8_t* allocate(Tindex index, size_t size) { + const auto maxSize = std::numeric_limits::max(); + if (size > maxSize) { + throw std::invalid_argument( + "requested "+std::to_string(size)+" bytes but limit is "+ + std::to_string(maxSize)); + } if (size == 0) { - throw std::runtime_error("size is 0"); + throw std::invalid_argument("zero size"); } ptrdiff_t offset = 0; if (auto found = find(index)) { diff --git a/test/data/StructLayout.cpp b/test/data/StructLayout.cpp index bf6f8e1d..ce183a11 100644 --- a/test/data/StructLayout.cpp +++ b/test/data/StructLayout.cpp @@ -114,3 +114,20 @@ TEST(StructLayout, ConvertWithLoss) { EXPECT_EQ(dstLayout.getChars(dst, "text"), "tru"); EXPECT_EQ(dstLayout.getInteger(dst, "someint"), INT8_MAX); } + +TEST(StructLayout, Serialization) { + std::vector fields { + Field {FieldType::CHAR, "text", 5}, + Field {FieldType::I16, "someint", 1}, + Field {FieldType::F64, "pi", 1}, + }; + auto layout1 = StructLayout::create(fields); + auto serialized = layout1.serialize(); + + std::cout << *serialized << std::endl; + + StructLayout layout2; + layout2.deserialize(serialized.get()); + + EXPECT_EQ(layout1, layout2); +} From 3d219b9a1ce581a14f02a2aa5a8ef40809f14b9e Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 14 Sep 2024 19:15:41 +0300 Subject: [PATCH 38/61] fix msvc build --- src/data/StructLayout.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index 35942457..c0d27229 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -15,7 +15,7 @@ namespace data { COUNT }; - inline const char* to_string(FieldType type) { + inline std::string to_string(FieldType type) { const char* names[] = { "int8", "int16", "int32", "int64", "float32", "float64", "char" }; From 44753904dcc84f706bcba92999561435f06dcdb4 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 14 Sep 2024 20:26:36 +0300 Subject: [PATCH 39/61] add Chunk.blocksMetadata --- src/voxels/Block.cpp | 3 +++ src/voxels/Block.hpp | 7 +++++++ src/voxels/Chunk.hpp | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/voxels/Block.cpp b/src/voxels/Block.cpp index ec125430..f06f052d 100644 --- a/src/voxels/Block.cpp +++ b/src/voxels/Block.cpp @@ -3,6 +3,7 @@ #include #include "core_defs.hpp" +#include "data/StructLayout.hpp" #include "util/stringutil.hpp" std::string to_string(BlockModel model) { @@ -102,6 +103,8 @@ Block::Block(const std::string& name) } { } +Block::~Block() {} + Block::Block(std::string name, const std::string& texture) : name(std::move(name)), textureFaces {texture, texture, texture, texture, texture, texture} { diff --git a/src/voxels/Block.hpp b/src/voxels/Block.hpp index 210972bb..4cbbe394 100644 --- a/src/voxels/Block.hpp +++ b/src/voxels/Block.hpp @@ -9,6 +9,10 @@ #include "maths/aabb.hpp" #include "typedefs.hpp" +namespace data { + class StructLayout; +} + inline std::string BLOCK_ITEM_SUFFIX = ".item"; inline constexpr uint FACE_MX = 0; @@ -184,6 +188,8 @@ public: // @brief Block tick interval (1 - 20tps, 2 - 10tps) uint tickInterval = 1; + std::unique_ptr dataStruct; + /// @brief Runtime indices (content indexing results) struct { /// @brief block runtime integer id @@ -211,6 +217,7 @@ public: Block(const std::string& name); Block(std::string name, const std::string& texture); Block(const Block&) = delete; + ~Block(); void cloneTo(Block& dst); }; diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index 79f75c28..1c537671 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -7,11 +7,11 @@ #include "constants.hpp" #include "lighting/Lightmap.hpp" +#include "util/SmallHeap.hpp" #include "voxel.hpp" inline constexpr int CHUNK_DATA_LEN = CHUNK_VOL * 4; -class Lightmap; class ContentReport; class Inventory; @@ -40,6 +40,8 @@ public: /// @brief Block inventories map where key is index of block in voxels array chunk_inventories_map inventories; + /// @brief Blocks metadata heap + util::SmallHeap blocksMetadata; Chunk(int x, int z); From fc99343fb581d66bc48a9e89a68ad423edea938c Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 30 Sep 2024 18:55:01 +0300 Subject: [PATCH 40/61] add block.get_field(...), block.set_field(...) --- src/content/ContentLoader.cpp | 7 ++ src/data/StructLayout.cpp | 26 +++---- src/data/StructLayout.hpp | 51 ++++++++++--- src/logic/scripting/lua/libblock.cpp | 105 +++++++++++++++++++++++++++ src/voxels/Chunks.cpp | 8 ++ 5 files changed, 169 insertions(+), 28 deletions(-) diff --git a/src/content/ContentLoader.cpp b/src/content/ContentLoader.cpp index 5c10b6e3..33b0cf4a 100644 --- a/src/content/ContentLoader.cpp +++ b/src/content/ContentLoader.cpp @@ -21,6 +21,7 @@ #include "util/stringutil.hpp" #include "voxels/Block.hpp" #include "data/dv_util.hpp" +#include "data/StructLayout.hpp" namespace fs = std::filesystem; @@ -250,6 +251,12 @@ void ContentLoader::loadBlock( root.at("ui-layout").get(def.uiLayout); root.at("inventory-size").get(def.inventorySize); root.at("tick-interval").get(def.tickInterval); + + if (root.has("fields")) { + def.dataStruct = std::make_unique(); + def.dataStruct->deserialize(root["fields"]); + } + if (def.tickInterval == 0) { def.tickInterval = 1; } diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp index 554dd0dc..9007213c 100644 --- a/src/data/StructLayout.cpp +++ b/src/data/StructLayout.cpp @@ -183,7 +183,7 @@ std::vector StructLayout::checkCompatibility( return report; } -const Field& StructLayout::requreField(const std::string& name) const { +const Field& StructLayout::requireField(const std::string& name) const { auto found = indices.find(name); if (found == indices.end()) { throw std::runtime_error("field '"+name+"' does not exist"); @@ -200,9 +200,8 @@ static void set_int(ubyte* dst, integer_t value) { } void StructLayout::setInteger( - ubyte* dst, integer_t value, const std::string& name, int index + ubyte* dst, integer_t value, const Field& field, int index ) const { - const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); @@ -216,7 +215,7 @@ void StructLayout::setInteger( case FieldType::CHAR: set_int(ptr, value); break; case FieldType::F32: case FieldType::F64: - setNumber(dst, static_cast(value), name, index); + setNumber(dst, static_cast(value), field, index); break; default: throw std::runtime_error("type error"); @@ -224,9 +223,8 @@ void StructLayout::setInteger( } void StructLayout::setNumber( - ubyte* dst, number_t value, const std::string& name, int index + ubyte* dst, number_t value, const Field& field, int index ) const { - const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); @@ -255,7 +253,7 @@ void StructLayout::setNumber( size_t StructLayout::setAscii( ubyte* dst, std::string_view value, const std::string& name ) const { - const auto& field = requreField(name); + const auto& field = requireField(name); if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); } @@ -266,9 +264,8 @@ size_t StructLayout::setAscii( } size_t StructLayout::setUnicode( - ubyte* dst, std::string_view value, const std::string& name + ubyte* dst, std::string_view value, const Field& field ) const { - const auto& field = requreField(name); if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); } @@ -285,9 +282,8 @@ static T get_int(const ubyte* src) { } integer_t StructLayout::getInteger( - const ubyte* src, const std::string& name, int index + const ubyte* src, const Field& field, int index ) const { - const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); @@ -305,9 +301,8 @@ integer_t StructLayout::getInteger( } number_t StructLayout::getNumber( - const ubyte* src, const std::string& name, int index + const ubyte* src, const Field& field, int index ) const { - const auto& field = requreField(name); if (index < 0 || index >= field.elements) { throw std::out_of_range( "index out of bounds [0, "+std::to_string(field.elements)+"]"); @@ -331,16 +326,15 @@ number_t StructLayout::getNumber( case FieldType::I32: case FieldType::I64: case FieldType::CHAR: - return getInteger(src, name, index); + return getInteger(src, field, index); default: throw std::runtime_error("type error"); } } std::string_view StructLayout::getChars( - const ubyte* src, const std::string& name + const ubyte* src, const Field& field ) const { - const auto& field = requreField(name); if (field.type != FieldType::CHAR) { throw std::runtime_error("'char' field type required"); } diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index f94d9365..4e0ff886 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -129,35 +129,54 @@ namespace data { /// @throws std::runtime_exception - field not found /// @param name field name /// @return read-only field reference - const Field& requreField(const std::string& name) const; + const Field& requireField(const std::string& name) const; + + [[nodiscard]] + integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const { + return getInteger(src, requireField(name), index); + } /// @brief Get integer from specified field. /// Types: (i8, i16, i32, i64, char) /// @throws std::runtime_exception - field not found /// @throws std::out_of_range - index is out of range [0, elements-1] /// @param src source buffer - /// @param name field name + /// @param field target field /// @param index array index /// @return field value [[nodiscard]] - integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const; + integer_t getInteger(const ubyte* src, const Field& field, int index=0) const; + + [[nodiscard]] + number_t getNumber(const ubyte* src, const std::string& name, int index=0) const { + return getNumber(src, requireField(name), index); + } /// @brief Get floating-point number from specified field. /// Types: (f32, f64, i8, i16, i32, i64, char) /// @throws std::runtime_exception - field not found /// @throws std::out_of_range - index is out of range [0, elements-1] /// @param src source buffer - /// @param name field name + /// @param field target field /// @param index array index /// @return field value [[nodiscard]] - number_t getNumber(const ubyte* src, const std::string& name, int index=0) const; + number_t getNumber(const ubyte* src, const Field& field, int index=0) const; + + [[nodiscard]] + std::string_view getChars(const ubyte* src, const std::string& name) const { + return getChars(src, requireField(name)); + } /// @brief Get read-only chars array as string_view. /// @param src source buffer - /// @param name field name + /// @param field target field [[nodiscard]] - std::string_view getChars(const ubyte* src, const std::string& name) const; + std::string_view getChars(const ubyte* src, const Field& field) const; + + void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0) const { + setInteger(dst, value, requireField(name), index); + } /// @brief Set field integer value. /// Types: (i8, i16, i32, i64, f32, f64, char) @@ -165,9 +184,13 @@ namespace data { /// @throws std::out_of_range - index is out of range [0, elements-1] /// @param dst destination buffer /// @param value value - /// @param name field name + /// @param field target field /// @param index array index - void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0) const; + void setInteger(ubyte* dst, integer_t value, const Field& field, int index=0) const; + + void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0) const { + setNumber(dst, value, requireField(name), index); + } /// @brief Set field numeric value. /// Types: (f32, f64) @@ -175,9 +198,9 @@ namespace data { /// @throws std::out_of_range - index is out of range [0, elements-1] /// @param dst destination buffer /// @param value value - /// @param name field name + /// @param field target field /// @param index array index - void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0) const; + void setNumber(ubyte* dst, number_t value, const Field& field, int index=0) const; /// @brief Replace chars array to given ASCII string /// @throws std::runtime_exception - field not found @@ -187,6 +210,10 @@ namespace data { /// @param name field name /// @return number of written string chars size_t setAscii(ubyte* dst, std::string_view value, const std::string& name) const; + + size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name) const { + return setUnicode(dst, value, requireField(name)); + } /// @brief Unicode-safe version of setAscii /// @throws std::runtime_exception - field not found @@ -194,7 +221,7 @@ namespace data { /// @param value utf-8 string /// @param name field name /// @return number of written string chars - size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name) const; + size_t setUnicode(ubyte* dst, std::string_view value, const Field& field) const; /// @return total structure size (bytes) [[nodiscard]] size_t size() const { diff --git a/src/logic/scripting/lua/libblock.cpp b/src/logic/scripting/lua/libblock.cpp index bd906f5d..f282422c 100644 --- a/src/logic/scripting/lua/libblock.cpp +++ b/src/logic/scripting/lua/libblock.cpp @@ -7,6 +7,8 @@ #include "voxels/Chunks.hpp" #include "voxels/voxel.hpp" #include "world/Level.hpp" +#include "maths/voxmaths.hpp" +#include "data/StructLayout.hpp" #include "api_lua.hpp" using namespace scripting; @@ -436,6 +438,107 @@ static int l_decompose_state(lua::State* L) { return 1; } +static int l_get_field(lua::State* L) { + auto x = lua::tointeger(L, 1); + auto y = lua::tointeger(L, 2); + auto z = lua::tointeger(L, 3); + auto name = lua::require_string(L, 4); + size_t index = 0; + if (lua::gettop(L) >= 5) { + index = lua::tointeger(L, 5); + } + auto vox = level->chunks->get(x, y, z); + auto cx = floordiv(x, CHUNK_W); + auto cz = floordiv(z, CHUNK_D); + auto chunk = level->chunks->getChunk(cx, cz); + auto lx = x - cx * CHUNK_W; + auto lz = z - cz * CHUNK_W; + size_t voxelIndex = vox_index(lx, y, lz); + + const auto& def = content->getIndices()->blocks.require(vox->id); + if (def.dataStruct == nullptr) { + return 0; + } + const auto& dataStruct = *def.dataStruct; + const auto field = dataStruct.getField(name); + if (field == nullptr) { + return 0; + } + const ubyte* src = chunk->blocksMetadata.find(voxelIndex); + if (src == nullptr) { + throw std::runtime_error("block data is not allocated"); + } + switch (field->type) { + case data::FieldType::I8: + case data::FieldType::I16: + case data::FieldType::I32: + case data::FieldType::I64: + return lua::pushinteger(L, dataStruct.getInteger(src, *field, index)); + case data::FieldType::F32: + case data::FieldType::F64: + return lua::pushnumber(L, dataStruct.getNumber(src, *field, index)); + case data::FieldType::CHAR: + return lua::pushstring(L, + std::string(dataStruct.getChars(src, *field)).c_str()); + case data::FieldType::COUNT: + return 0; + } + return 0; +} + +static int l_set_field(lua::State* L) { + auto x = lua::tointeger(L, 1); + auto y = lua::tointeger(L, 2); + auto z = lua::tointeger(L, 3); + auto name = lua::require_string(L, 4); + auto value = lua::tovalue(L, 5); + size_t index = 0; + if (lua::gettop(L) >= 6) { + index = lua::tointeger(L, 6); + } + auto vox = level->chunks->get(x, y, z); + auto cx = floordiv(x, CHUNK_W); + auto cz = floordiv(z, CHUNK_D); + auto chunk = level->chunks->getChunk(cx, cz); + auto lx = x - cx * CHUNK_W; + auto lz = z - cz * CHUNK_W; + size_t voxelIndex = vox_index(lx, y, lz); + + const auto& def = content->getIndices()->blocks.require(vox->id); + if (def.dataStruct == nullptr) { + return 0; + } + const auto& dataStruct = *def.dataStruct; + const auto field = dataStruct.getField(name); + if (field == nullptr) { + return 0; + } + ubyte* dst = chunk->blocksMetadata.find(voxelIndex); + if (dst == nullptr) { + throw std::runtime_error("block data is not allocated"); + } + switch (field->type) { + case data::FieldType::CHAR: + if (value.isString()) { + return lua::pushinteger(L, + dataStruct.setUnicode(dst, value.asString(), *field)); + } + case data::FieldType::I8: + case data::FieldType::I16: + case data::FieldType::I32: + case data::FieldType::I64: + dataStruct.setInteger(dst, value.asInteger(), *field, index); + break; + case data::FieldType::F32: + case data::FieldType::F64: + dataStruct.setNumber(dst, value.asNumber(), *field, index); + break; + case data::FieldType::COUNT: + break; + } + return 0; +} + const luaL_Reg blocklib[] = { {"index", lua::wrap}, {"name", lua::wrap}, @@ -469,5 +572,7 @@ const luaL_Reg blocklib[] = { {"raycast", lua::wrap}, {"compose_state", lua::wrap}, {"decompose_state", lua::wrap}, + {"get_field", lua::wrap}, + {"set_field", lua::wrap}, {NULL, NULL} }; diff --git a/src/voxels/Chunks.cpp b/src/voxels/Chunks.cpp index 0e194dce..9b584007 100644 --- a/src/voxels/Chunks.cpp +++ b/src/voxels/Chunks.cpp @@ -6,6 +6,7 @@ #include #include +#include "data/StructLayout.hpp" #include "coders/byte_utils.hpp" #include "coders/json.hpp" #include "content/Content.hpp" @@ -370,6 +371,7 @@ void Chunks::set( } int lx = x - cx * CHUNK_W; int lz = z - cz * CHUNK_D; + size_t index = vox_index(lx, y, lz); // block finalization voxel& vox = chunk->voxels[(y * CHUNK_D + lz) * CHUNK_W + lx]; @@ -380,6 +382,9 @@ void Chunks::set( if (prevdef.rt.extended && !vox.state.segment) { eraseSegments(prevdef, vox.state, gx, y, gz); } + if (prevdef.dataStruct) { + chunk->blocksMetadata.free(chunk->blocksMetadata.find(index)); + } // block initialization const auto& newdef = indices->blocks.require(id); @@ -389,6 +394,9 @@ void Chunks::set( if (!state.segment && newdef.rt.extended) { repairSegments(newdef, state, gx, y, gz); } + if (newdef.dataStruct) { + chunk->blocksMetadata.allocate(index, newdef.dataStruct->size()); + } if (y < chunk->bottom) chunk->bottom = y; From 4f559c108617361d6b999efa4de01589399bfb5c Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 30 Sep 2024 20:15:22 +0300 Subject: [PATCH 41/61] refactor libblock --- src/logic/scripting/lua/libblock.cpp | 75 ++++++++++++++++++---------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/src/logic/scripting/lua/libblock.cpp b/src/logic/scripting/lua/libblock.cpp index f282422c..16b924f1 100644 --- a/src/logic/scripting/lua/libblock.cpp +++ b/src/logic/scripting/lua/libblock.cpp @@ -438,6 +438,31 @@ static int l_decompose_state(lua::State* L) { return 1; } +static int get_field( + lua::State* L, + const ubyte* src, + const data::Field& field, + size_t index, + const data::StructLayout& dataStruct +) { + switch (field.type) { + case data::FieldType::I8: + case data::FieldType::I16: + case data::FieldType::I32: + case data::FieldType::I64: + return lua::pushinteger(L, dataStruct.getInteger(src, field, index)); + case data::FieldType::F32: + case data::FieldType::F64: + return lua::pushnumber(L, dataStruct.getNumber(src, field, index)); + case data::FieldType::CHAR: + return lua::pushstring(L, + std::string(dataStruct.getChars(src, field)).c_str()); + case data::FieldType::COUNT: + return 0; + } + return 0; +} + static int l_get_field(lua::State* L) { auto x = lua::tointeger(L, 1); auto y = lua::tointeger(L, 2); @@ -468,20 +493,35 @@ static int l_get_field(lua::State* L) { if (src == nullptr) { throw std::runtime_error("block data is not allocated"); } - switch (field->type) { + return get_field(L, src, *field, index, dataStruct); +} + +static int set_field( + lua::State* L, + ubyte* dst, + const data::Field& field, + size_t index, + const data::StructLayout& dataStruct, + const dv::value& value +) { + switch (field.type) { + case data::FieldType::CHAR: + if (value.isString()) { + return lua::pushinteger(L, + dataStruct.setUnicode(dst, value.asString(), field)); + } case data::FieldType::I8: case data::FieldType::I16: case data::FieldType::I32: case data::FieldType::I64: - return lua::pushinteger(L, dataStruct.getInteger(src, *field, index)); + dataStruct.setInteger(dst, value.asInteger(), field, index); + break; case data::FieldType::F32: case data::FieldType::F64: - return lua::pushnumber(L, dataStruct.getNumber(src, *field, index)); - case data::FieldType::CHAR: - return lua::pushstring(L, - std::string(dataStruct.getChars(src, *field)).c_str()); + dataStruct.setNumber(dst, value.asNumber(), field, index); + break; case data::FieldType::COUNT: - return 0; + break; } return 0; } @@ -517,26 +557,7 @@ static int l_set_field(lua::State* L) { if (dst == nullptr) { throw std::runtime_error("block data is not allocated"); } - switch (field->type) { - case data::FieldType::CHAR: - if (value.isString()) { - return lua::pushinteger(L, - dataStruct.setUnicode(dst, value.asString(), *field)); - } - case data::FieldType::I8: - case data::FieldType::I16: - case data::FieldType::I32: - case data::FieldType::I64: - dataStruct.setInteger(dst, value.asInteger(), *field, index); - break; - case data::FieldType::F32: - case data::FieldType::F64: - dataStruct.setNumber(dst, value.asNumber(), *field, index); - break; - case data::FieldType::COUNT: - break; - } - return 0; + return set_field(L, dst, *field, index, dataStruct, value); } const luaL_Reg blocklib[] = { From e84c79839ca75d4d646123c147316fbad944fa67 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 30 Sep 2024 21:33:19 +0300 Subject: [PATCH 42/61] update blocks data lifetime --- src/logic/scripting/lua/libblock.cpp | 4 ++-- src/voxels/Chunks.cpp | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/logic/scripting/lua/libblock.cpp b/src/logic/scripting/lua/libblock.cpp index 16b924f1..ea31e446 100644 --- a/src/logic/scripting/lua/libblock.cpp +++ b/src/logic/scripting/lua/libblock.cpp @@ -491,7 +491,7 @@ static int l_get_field(lua::State* L) { } const ubyte* src = chunk->blocksMetadata.find(voxelIndex); if (src == nullptr) { - throw std::runtime_error("block data is not allocated"); + return 0; } return get_field(L, src, *field, index, dataStruct); } @@ -555,7 +555,7 @@ static int l_set_field(lua::State* L) { } ubyte* dst = chunk->blocksMetadata.find(voxelIndex); if (dst == nullptr) { - throw std::runtime_error("block data is not allocated"); + dst = chunk->blocksMetadata.allocate(voxelIndex, dataStruct.size()); } return set_field(L, dst, *field, index, dataStruct, value); } diff --git a/src/voxels/Chunks.cpp b/src/voxels/Chunks.cpp index 9b584007..408ab828 100644 --- a/src/voxels/Chunks.cpp +++ b/src/voxels/Chunks.cpp @@ -394,9 +394,6 @@ void Chunks::set( if (!state.segment && newdef.rt.extended) { repairSegments(newdef, state, gx, y, gz); } - if (newdef.dataStruct) { - chunk->blocksMetadata.allocate(index, newdef.dataStruct->size()); - } if (y < chunk->bottom) chunk->bottom = y; From 28d746f3718956899516651aaebcdd640c449f44 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 30 Sep 2024 23:42:16 +0300 Subject: [PATCH 43/61] implement blocks data saving/loading --- src/files/WorldRegions.cpp | 24 ++++++++++++++ src/files/WorldRegions.hpp | 2 ++ src/files/compatibility.cpp | 6 ++-- src/files/world_regions_fwd.hpp | 1 + src/logic/scripting/lua/libblock.cpp | 2 ++ src/util/Buffer.hpp | 10 +++++- src/util/SmallHeap.hpp | 49 +++++++++++++++++++++++----- src/voxels/Chunk.hpp | 5 ++- src/voxels/Chunks.cpp | 6 +++- src/voxels/ChunksStorage.cpp | 1 + test/util/SmallHeap.cpp | 16 +++++++++ 11 files changed, 107 insertions(+), 15 deletions(-) diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index d42aa072..35d15ac7 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -68,6 +68,9 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { layers[REGION_LAYER_INVENTORIES].folder = directory / fs::path("inventories"); layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities"); + + auto& blocksData = layers[REGION_LAYER_BLOCKS_DATA]; + blocksData.folder = directory / fs::path("blocksdata"); } WorldRegions::~WorldRegions() = default; @@ -188,6 +191,15 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { std::move(data), entitiesData.size()); } + // Writing blocks data + if (chunk->flags.blocksData) { + auto bytes = chunk->blocksMetadata.serialize(); + put(chunk->x, + chunk->z, + REGION_LAYER_BLOCKS_DATA, + bytes.release(), + bytes.size()); + } } std::unique_ptr WorldRegions::getVoxels(int x, int z) { @@ -227,6 +239,18 @@ chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { return load_inventories(bytes, bytesSize); } +BlocksMetadata WorldRegions::getBlocksData(int x, int z) { + uint32_t bytesSize; + uint32_t srcSize; + auto bytes = layers[REGION_LAYER_BLOCKS_DATA].getData(x, z, bytesSize, srcSize); + if (bytes == nullptr) { + return {}; + } + BlocksMetadata heap; + heap.deserialize(bytes, bytesSize); + return heap; +} + void WorldRegions::processInventories( int x, int z, const inventoryproc& func ) { diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index ff548b6f..4fe37ab7 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -216,6 +216,8 @@ public: std::unique_ptr getLights(int x, int z); chunk_inventories_map fetchInventories(int x, int z); + + BlocksMetadata getBlocksData(int x, int z); /// @brief Load saved entities data for chunk /// @param x chunk.x diff --git a/src/files/compatibility.cpp b/src/files/compatibility.cpp index 8cdd437d..0ffc97fe 100644 --- a/src/files/compatibility.cpp +++ b/src/files/compatibility.cpp @@ -12,7 +12,6 @@ static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4; static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4; -#include static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t size) { auto data = compression::decompress( buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8); @@ -40,11 +39,9 @@ static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t siz return util::Buffer(std::move(compressed), outLen); } -#include "util/timeutil.hpp" util::Buffer compatibility::convert_region_2to3( const util::Buffer& src, RegionLayerIndex layer ) { - timeutil::ScopeLogTimer log(555); const size_t REGION_CHUNKS = 1024; const size_t HEADER_SIZE = 10; const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t); @@ -95,7 +92,8 @@ util::Buffer compatibility::convert_region_2to3( builder.put(data, size); break; case REGION_LAYER_ENTITIES: - case REGION_LAYER_INVENTORIES: { + case REGION_LAYER_INVENTORIES: + case REGION_LAYER_BLOCKS_DATA: { builder.putInt32(size); builder.putInt32(size); builder.put(data, size); diff --git a/src/files/world_regions_fwd.hpp b/src/files/world_regions_fwd.hpp index e0398641..9938239d 100644 --- a/src/files/world_regions_fwd.hpp +++ b/src/files/world_regions_fwd.hpp @@ -7,6 +7,7 @@ enum RegionLayerIndex : uint { REGION_LAYER_LIGHTS, REGION_LAYER_INVENTORIES, REGION_LAYER_ENTITIES, + REGION_LAYER_BLOCKS_DATA, REGION_LAYERS_COUNT }; diff --git a/src/logic/scripting/lua/libblock.cpp b/src/logic/scripting/lua/libblock.cpp index ea31e446..f186fdae 100644 --- a/src/logic/scripting/lua/libblock.cpp +++ b/src/logic/scripting/lua/libblock.cpp @@ -557,6 +557,8 @@ static int l_set_field(lua::State* L) { if (dst == nullptr) { dst = chunk->blocksMetadata.allocate(voxelIndex, dataStruct.size()); } + chunk->flags.unsaved = true; + chunk->flags.blocksData = true; return set_field(L, dst, *field, index, dataStruct, value); } diff --git a/src/util/Buffer.hpp b/src/util/Buffer.hpp index 01769f0d..fbe1ade7 100644 --- a/src/util/Buffer.hpp +++ b/src/util/Buffer.hpp @@ -4,6 +4,8 @@ #include namespace util { + /// @brief Template similar to std::unique_ptr stores a buffer with its size + /// @tparam T buffer elements type template class Buffer { std::unique_ptr ptr; @@ -12,7 +14,9 @@ namespace util { Buffer(size_t length) : ptr(std::make_unique(length)), length(length) { } - Buffer(const Buffer& o) : Buffer(o.data(), o.size()) {} + explicit Buffer(const Buffer& o) : Buffer(o.data(), o.size()) {} + + Buffer(Buffer&& o) : ptr(std::move(o.ptr)), length(o.length) {} Buffer(std::unique_ptr ptr, size_t length) : ptr(std::move(ptr)), length(length) {} @@ -42,14 +46,18 @@ namespace util { return length; } + /// @brief Take ownership over the buffer unique_ptr std::unique_ptr release() { return std::move(ptr); } + /// @brief Create a buffer copy Buffer clone() const { return Buffer(ptr.get(), length); } + /// @brief Update buffer size without releasing used memory + /// @param size new size (must be less or equal to current) void resizeFast(size_t size) { length = size; } diff --git a/src/util/SmallHeap.hpp b/src/util/SmallHeap.hpp index 0279f465..189d54c2 100644 --- a/src/util/SmallHeap.hpp +++ b/src/util/SmallHeap.hpp @@ -6,7 +6,15 @@ #include #include -namespace util { +#include "Buffer.hpp" +#include "data_io.hpp" + +namespace util { + template + inline T read_int_le(const uint8_t* src, size_t offset=0) { + return dataio::le2h(*(reinterpret_cast(src) + offset)); + } + // TODO: make it safer (minimize raw temporary pointers use) /// @brief Simple heap implementation for memory-optimal sparse array of /// small different structures @@ -28,9 +36,9 @@ namespace util { uint8_t* find(Tindex index) { auto data = buffer.data(); for (size_t i = 0; i < entriesCount; i++) { - auto nextIndex = *reinterpret_cast(data); + auto nextIndex = read_int_le(data); data += sizeof(Tindex); - auto nextSize = *reinterpret_cast(data); + auto nextSize = read_int_le(data); data += sizeof(Tsize); if (nextIndex == index) { return data; @@ -85,9 +93,9 @@ namespace util { } for (size_t i = 0; i < entriesCount; i++) { auto data = buffer.data() + offset; - auto nextIndex = *reinterpret_cast(data); + auto nextIndex = read_int_le(data); data += sizeof(Tindex); - auto nextSize = *reinterpret_cast(data); + auto nextSize = read_int_le(data); data += sizeof(Tsize); if (nextIndex > index) { break; @@ -103,9 +111,9 @@ namespace util { entriesCount++; auto data = buffer.data() + offset; - *reinterpret_cast(data) = index; + *reinterpret_cast(data) = dataio::h2le(index); data += sizeof(Tindex); - *reinterpret_cast(data) = size; + *reinterpret_cast(data) = dataio::h2le(size); return data + sizeof(Tsize); } @@ -115,7 +123,7 @@ namespace util { if (ptr == nullptr) { return 0; } - return *(reinterpret_cast(ptr)-1); + return read_int_le(ptr, -1); } /// @return number of entries @@ -127,5 +135,30 @@ namespace util { size_t size() const { return buffer.size(); } + + inline bool operator==(const SmallHeap& o) const { + if (o.entriesCount != entriesCount) { + return false; + } + return buffer == o.buffer; + } + + util::Buffer serialize() const { + util::Buffer out(sizeof(Tindex) + buffer.size()); + ubyte* dst = out.data(); + const ubyte* src = buffer.data(); + + *reinterpret_cast(dst) = dataio::h2le(entriesCount); + dst += sizeof(Tindex); + + std::memcpy(dst, src, buffer.size()); + return out; + } + + void deserialize(const ubyte* src, size_t size) { + entriesCount = read_int_le(src); + buffer.resize(size - sizeof(Tindex)); + std::memcpy(buffer.data(), src + sizeof(Tindex), buffer.size()); + } }; } diff --git a/src/voxels/Chunk.hpp b/src/voxels/Chunk.hpp index 184fb2d9..d7a6979a 100644 --- a/src/voxels/Chunk.hpp +++ b/src/voxels/Chunk.hpp @@ -18,6 +18,8 @@ class Inventory; using chunk_inventories_map = std::unordered_map>; +using BlocksMetadata = util::SmallHeap; + class Chunk { public: int x, z; @@ -32,12 +34,13 @@ public: bool unsaved : 1; bool loadedLights : 1; bool entities : 1; + bool blocksData : 1; } flags {}; /// @brief Block inventories map where key is index of block in voxels array chunk_inventories_map inventories; /// @brief Blocks metadata heap - util::SmallHeap blocksMetadata; + BlocksMetadata blocksMetadata; Chunk(int x, int z); diff --git a/src/voxels/Chunks.cpp b/src/voxels/Chunks.cpp index 408ab828..7196670f 100644 --- a/src/voxels/Chunks.cpp +++ b/src/voxels/Chunks.cpp @@ -383,7 +383,11 @@ void Chunks::set( eraseSegments(prevdef, vox.state, gx, y, gz); } if (prevdef.dataStruct) { - chunk->blocksMetadata.free(chunk->blocksMetadata.find(index)); + if (auto found = chunk->blocksMetadata.find(index)) { + chunk->blocksMetadata.free(found); + chunk->flags.unsaved = true; + chunk->flags.blocksData = true; + } } // block initialization diff --git a/src/voxels/ChunksStorage.cpp b/src/voxels/ChunksStorage.cpp index af8127f4..36a98ea7 100644 --- a/src/voxels/ChunksStorage.cpp +++ b/src/voxels/ChunksStorage.cpp @@ -81,6 +81,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { chunk->lightmap.set(lights.get()); chunk->flags.loadedLights = true; } + chunk->blocksMetadata = regions.getBlocksData(chunk->x, chunk->z); return chunk; } diff --git a/test/util/SmallHeap.cpp b/test/util/SmallHeap.cpp index d8b08a4c..63af3592 100644 --- a/test/util/SmallHeap.cpp +++ b/test/util/SmallHeap.cpp @@ -53,3 +53,19 @@ TEST(SmallHeap, RandomFill) { } EXPECT_EQ(map.sizeOf(map.find(n)), 123); } + +TEST(SmallHeap, EncodeDecode) { + SmallHeap map; + int n = 3'000; + map.allocate(n, 123); + for (int i = 0; i < n; i++) { + int index = rand() % n; + int size = rand() % 254 + 1; + map.allocate(index, size); + } + auto bytes = map.serialize(); + + SmallHeap out; + out.deserialize(bytes.data(), bytes.size()); + EXPECT_EQ(map, out); +} From bc05716772e0ad58b7e6d69cd56ad65fa02aaaba Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 1 Oct 2024 19:08:45 +0300 Subject: [PATCH 44/61] implement blocks data conversion --- res/texts/en_US.txt | 1 + res/texts/ru_RU.txt | 1 + src/content/ContentReport.cpp | 31 ++++++++++++- src/content/ContentReport.hpp | 12 +++++ src/data/StructLayout.hpp | 6 +++ src/files/WorldConverter.cpp | 51 ++++++++++++++++++++- src/files/WorldConverter.hpp | 3 ++ src/files/WorldFiles.cpp | 10 +++++ src/files/WorldRegions.cpp | 82 ++++++++++++++++++++++++++++++---- src/files/WorldRegions.hpp | 16 ++++--- src/graphics/ui/gui_util.cpp | 45 +++++++++++++++++++ src/graphics/ui/gui_util.hpp | 8 ++++ src/logic/EngineController.cpp | 32 ++++++++++--- src/util/SmallHeap.hpp | 52 +++++++++++++++++++++ src/voxels/ChunksStorage.cpp | 1 + test/util/SmallHeap.cpp | 13 ++++++ 16 files changed, 341 insertions(+), 23 deletions(-) diff --git a/res/texts/en_US.txt b/res/texts/en_US.txt index 9cf71339..41988abf 100644 --- a/res/texts/en_US.txt +++ b/res/texts/en_US.txt @@ -2,6 +2,7 @@ menu.missing-content=Missing Content! world.convert-request=Content indices have changed! Convert world files? world.upgrade-request=World format is outdated! Convert world files? +world.convert-with-loss=Convert world with data loss? pack.remove-confirm=Do you want to erase all pack(s) content from the world forever? error.pack-not-found=Could not to find pack error.dependency-not-found=Dependency pack is not found diff --git a/res/texts/ru_RU.txt b/res/texts/ru_RU.txt index 20021b53..2ea4610f 100644 --- a/res/texts/ru_RU.txt +++ b/res/texts/ru_RU.txt @@ -47,6 +47,7 @@ world.generators.flat=Плоский world.Create World=Создать Мир world.convert-request=Есть изменения в индексах! Конвертировать мир? world.upgrade-request=Формат мира устарел! Конвертировать мир? +world.convert-with-loss=Конвертировать мир с потерями? world.delete-confirm=Удалить мир безвозвратно? # Настройки diff --git a/src/content/ContentReport.cpp b/src/content/ContentReport.cpp index 2dd72259..257bd3b3 100644 --- a/src/content/ContentReport.cpp +++ b/src/content/ContentReport.cpp @@ -56,11 +56,40 @@ std::shared_ptr ContentReport::create( indices, blocks_c, items_c, regionsVersion); report->blocks.setup(blocklist, content->blocks); report->items.setup(itemlist, content->items); + + for (const auto& [name, map] : root["blocks-data"].asObject()) { + data::StructLayout layout; + layout.deserialize(map); + auto def = content->blocks.find(name); + if (def == nullptr) { + continue; + } + if (def->dataStruct == nullptr) { + ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE}; + report->issues.push_back(issue); + report->dataLoss.push_back(name+": discard data"); + continue; + } + auto incapatibility = layout.checkCompatibility(*def->dataStruct); + if (!incapatibility.empty()) { + ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE}; + report->issues.push_back(issue); + for (const auto& error : incapatibility) { + report->dataLoss.push_back( + "[" + name + "] field " + error.name + " - " + + data::to_string(error.type) + ); + } + } + report->blocksDataLayouts[name] = std::move(layout); + } + report->buildIssues(); if (report->isUpgradeRequired() || report->hasContentReorder() || - report->hasMissingContent()) { + report->hasMissingContent() || + report->hasDataLoss()) { return report; } else { return nullptr; diff --git a/src/content/ContentReport.hpp b/src/content/ContentReport.hpp index c5c0e087..1c7cb274 100644 --- a/src/content/ContentReport.hpp +++ b/src/content/ContentReport.hpp @@ -4,11 +4,13 @@ #include #include #include +#include #include "constants.hpp" #include "data/dv.hpp" #include "typedefs.hpp" #include "Content.hpp" +#include "data/StructLayout.hpp" #include "files/world_regions_fwd.hpp" namespace fs = std::filesystem; @@ -17,6 +19,7 @@ enum class ContentIssueType { REORDER, MISSING, REGION_FORMAT_UPDATE, + BLOCK_DATA_LAYOUTS_UPDATE, }; struct ContentIssue { @@ -121,7 +124,9 @@ public: ContentUnitLUT items; uint regionsVersion; + std::unordered_map blocksDataLayouts; std::vector issues; + std::vector dataLoss; ContentReport( const ContentIndices* indices, @@ -136,6 +141,10 @@ public: const Content* content ); + inline const std::vector& getDataLoss() const { + return dataLoss; + } + inline bool hasContentReorder() const { return blocks.hasContentReorder() || items.hasContentReorder(); } @@ -145,6 +154,9 @@ public: inline bool isUpgradeRequired() const { return regionsVersion < REGION_FORMAT_VERSION; } + inline bool hasDataLoss() const { + return !dataLoss.empty(); + } void buildIssues(); const std::vector& getIssues() const; diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp index 4e0ff886..784be27b 100644 --- a/src/data/StructLayout.hpp +++ b/src/data/StructLayout.hpp @@ -30,6 +30,12 @@ namespace data { TYPE_ERROR, MISSING, }; + inline const char* to_string(FieldIncapatibilityType type) { + const char* names[] = { + "none", "data_loss", "type_error", "missing" + }; + return names[static_cast(type)]; + } struct FieldIncapatibility { std::string name; diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index ca50bc6e..3bd9b7b4 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -13,6 +13,7 @@ #include "util/ThreadPool.hpp" #include "voxels/Chunk.hpp" #include "items/Inventory.hpp" +#include "voxels/Block.hpp" #include "WorldFiles.hpp" namespace fs = std::filesystem; @@ -85,6 +86,7 @@ void WorldConverter::createConvertTasks() { const auto& regions = wfile->getRegions(); for (auto& issue : report->getIssues()) { switch (issue.issueType) { + case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE: case ContentIssueType::REGION_FORMAT_UPDATE: break; case ContentIssueType::MISSING: @@ -111,8 +113,24 @@ WorldConverter::WorldConverter( { if (upgradeMode) { createUpgradeTasks(); - } else { + } else if (report->hasContentReorder()) { createConvertTasks(); + } else { + // blocks data conversion requires correct block indices + // so it must be done AFTER voxels conversion + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + switch (issue.issueType) { + case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE: + addRegionsTasks( + REGION_LAYER_BLOCKS_DATA, + ConvertTaskType::CONVERT_BLOCKS_DATA + ); + break; + default: + break; + } + } } } @@ -187,6 +205,34 @@ void WorldConverter::convertPlayer(const fs::path& file) const { files::write_json(file, map); } +void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const { + logger.info() << "converting blocks data"; + wfile->getRegions().processBlocksData(x, z, + [=](BlocksMetadata& heap, std::unique_ptr voxelsData) { + Chunk chunk(0, 0); + chunk.decode(voxelsData.get()); + + const auto& indices = content->getIndices()->blocks; + + BlocksMetadata newHeap; + for (const auto& entry : heap) { + size_t index = entry.index; + const auto& def = indices.require(chunk.voxels[index].id); + const auto& newStruct = *def.dataStruct; + const auto& found = report.blocksDataLayouts.find(def.name); + if (found == report.blocksDataLayouts.end()) { + logger.error() << "no previous fields layout found for block" + << def.name << " - discard"; + continue; + } + const auto& prevStruct = found->second; + uint8_t* dst = newHeap.allocate(index, newStruct.size()); + newStruct.convert(prevStruct, entry.data(), dst, true); + } + heap = std::move(newHeap); + }); +} + void WorldConverter::convert(const ConvertTask& task) const { if (!fs::is_regular_file(task.file)) return; @@ -203,6 +249,9 @@ void WorldConverter::convert(const ConvertTask& task) const { case ConvertTaskType::PLAYER: convertPlayer(task.file); break; + case ConvertTaskType::CONVERT_BLOCKS_DATA: + convertBlocksData(task.x, task.z, *report); + break; } } diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index fe9e35de..6331acb6 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -24,6 +24,8 @@ enum class ConvertTaskType { PLAYER, /// @brief refresh region file version UPGRADE_REGION, + /// @brief convert blocks data to updated layouts + CONVERT_BLOCKS_DATA, }; struct ConvertTask { @@ -49,6 +51,7 @@ class WorldConverter : public Task { void convertPlayer(const fs::path& file) const; void convertVoxels(const fs::path& file, int x, int z) const; void convertInventories(const fs::path& file, int x, int z) const; + void convertBlocksData(int x, int z, const ContentReport& report) const; void addRegionsTasks( RegionLayerIndex layerid, diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 748a3cbc..7f59e602 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -21,6 +21,7 @@ #include "objects/EntityDef.hpp" #include "objects/Player.hpp" #include "physics/Hitbox.hpp" +#include "data/StructLayout.hpp" #include "settings.hpp" #include "typedefs.hpp" #include "util/data_io.hpp" @@ -116,6 +117,15 @@ void WorldFiles::writeIndices(const ContentIndices* indices) { write_indices(indices->blocks, root.list("blocks")); write_indices(indices->items, root.list("items")); write_indices(indices->entities, root.list("entities")); + + auto& structsMap = root.object("blocks-data"); + for (const auto* def : indices->blocks.getIterable()) { + if (def->dataStruct == nullptr) { + continue; + } + structsMap[def->name] = def->dataStruct->serialize(); + } + files::write_json(getIndicesFile(), root); } diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 35d15ac7..87ee4351 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -4,6 +4,7 @@ #include #include +#include "debug/Logger.hpp" #include "coders/byte_utils.hpp" #include "coders/rle.hpp" #include "coders/binary_json.hpp" @@ -13,6 +14,8 @@ #define REGION_FORMAT_MAGIC ".VOXREG" +static debug::Logger logger("world-regions"); + WorldRegion::WorldRegion() : chunksData( std::make_unique[]>(REGION_CHUNKS_COUNT) @@ -95,15 +98,21 @@ void WorldRegions::put( ) { size_t size = srcSize; auto& layer = layers[layerid]; - if (layer.compression != compression::Method::NONE) { - data = compression::compress( - data.get(), size, size, layer.compression); - } int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); + + if (data == nullptr) { + region->put(localX, localZ, nullptr, 0, 0); + return; + } + + if (layer.compression != compression::Method::NONE) { + data = compression::compress( + data.get(), size, size, layer.compression); + } region->put(localX, localZ, std::move(data), size, srcSize); } @@ -251,9 +260,7 @@ BlocksMetadata WorldRegions::getBlocksData(int x, int z) { return heap; } -void WorldRegions::processInventories( - int x, int z, const inventoryproc& func -) { +void WorldRegions::processInventories(int x, int z, const InventoryProc& func) { processRegion(x, z, REGION_LAYER_INVENTORIES, [=](std::unique_ptr data, uint32_t* size) { auto inventories = load_inventories(data.get(), *size); @@ -264,6 +271,65 @@ void WorldRegions::processInventories( }); } +void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) { + auto& voxLayer = layers[REGION_LAYER_VOXELS]; + auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA]; + if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) { + throw std::runtime_error("not implemented for in-memory regions"); + } + auto datRegfile = datLayer.getRegFile({x, z}); + if (datRegfile == nullptr) { + throw std::runtime_error("could not open region file"); + } + auto voxRegfile = voxLayer.getRegFile({x, z}); + if (voxRegfile == nullptr) { + logger.warning() << "missing voxels region - discard blocks data for " + << x << "_" << z; + abort(); // TODO: delete region file + } + for (uint cz = 0; cz < REGION_SIZE; cz++) { + for (uint cx = 0; cx < REGION_SIZE; cx++) { + int gx = cx + x * REGION_SIZE; + int gz = cz + z * REGION_SIZE; + + uint32_t datLength; + uint32_t datSrcSize; + auto datData = RegionsLayer::readChunkData( + gx, gz, datLength, datSrcSize, datRegfile.get() + ); + if (datData == nullptr) { + continue; + } + uint32_t voxLength; + uint32_t voxSrcSize; + auto voxData = RegionsLayer::readChunkData( + gx, gz, voxLength, voxSrcSize, voxRegfile.get() + ); + if (voxData == nullptr) { + logger.warning() + << "missing voxels for chunk (" << gx << ", " << gz << ")"; + put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0); + continue; + } + voxData = compression::decompress( + voxData.get(), voxLength, voxSrcSize, voxLayer.compression + ); + + BlocksMetadata blocksData; + blocksData.deserialize(datData.get(), datLength); + try { + func(blocksData, std::move(voxData)); + } catch (const std::exception& err) { + logger.error() << "an error ocurred while processing blocks " + "data in chunk (" << gx << ", " << gz << "): " << err.what(); + blocksData = {}; + } + auto bytes = blocksData.serialize(); + put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size()); + } + } +} + dv::value WorldRegions::fetchEntities(int x, int z) { if (generatorTestMode) { return nullptr; @@ -282,7 +348,7 @@ dv::value WorldRegions::fetchEntities(int x, int z) { } void WorldRegions::processRegion( - int x, int z, RegionLayerIndex layerid, const regionproc& func + int x, int z, RegionLayerIndex layerid, const RegionProc& func ) { auto& layer = layers[layerid]; if (layer.getRegion(x, z)) { diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index 4fe37ab7..722e5c48 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -64,9 +64,10 @@ struct regfile { std::unique_ptr read(int index, uint32_t& size, uint32_t& srcSize); }; -using regionsmap = std::unordered_map>; -using regionproc = std::function(std::unique_ptr,uint32_t*)>; -using inventoryproc = std::function; +using RegionsMap = std::unordered_map>; +using RegionProc = std::function(std::unique_ptr,uint32_t*)>; +using InventoryProc = std::function; +using BlockDataProc = std::function)>; /// @brief Region file pointer keeping inUse flag on until destroyed class regfile_ptr { @@ -125,7 +126,7 @@ struct RegionsLayer { compression::Method compression = compression::Method::NONE; /// @brief In-memory regions data - regionsmap regions; + RegionsMap regions; /// @brief In-memory regions map mutex std::mutex mapMutex; @@ -231,10 +232,11 @@ public: /// @param layerid regions layer index /// @param func processing callback void processRegion( - int x, int z, RegionLayerIndex layerid, const regionproc& func); + int x, int z, RegionLayerIndex layerid, const RegionProc& func); - void processInventories( - int x, int z, const inventoryproc& func); + void processInventories(int x, int z, const InventoryProc& func); + + void processBlocksData(int x, int z, const BlockDataProc& func); /// @brief Get regions directory by layer index /// @param layerid layer index diff --git a/src/graphics/ui/gui_util.cpp b/src/graphics/ui/gui_util.cpp index 2045713a..70bd94f5 100644 --- a/src/graphics/ui/gui_util.cpp +++ b/src/graphics/ui/gui_util.cpp @@ -3,6 +3,7 @@ #include "elements/Label.hpp" #include "elements/Menu.hpp" #include "elements/Button.hpp" +#include "elements/TextBox.hpp" #include "gui_xml.hpp" #include "logic/scripting/scripting.hpp" @@ -77,3 +78,47 @@ void guiutil::confirm( menu->addPage("", panel); menu->setPage(""); } + +void guiutil::confirmWithMemo( + gui::GUI* gui, + const std::wstring& text, + const std::wstring& memo, + const runnable& on_confirm, + std::wstring yestext, + std::wstring notext) { + + if (yestext.empty()) yestext = langs::get(L"Yes"); + if (notext.empty()) notext = langs::get(L"No"); + + auto menu = gui->getMenu(); + auto panel = std::make_shared(glm::vec2(600, 500), glm::vec4(8.0f), 8.0f); + panel->setColor(glm::vec4(0.0f, 0.0f, 0.0f, 0.5f)); + panel->add(std::make_shared