diff --git a/doc/en/block-properties.md b/doc/en/block-properties.md index 1827a026..cef58a9a 100644 --- a/doc/en/block-properties.md +++ b/doc/en/block-properties.md @@ -142,3 +142,42 @@ Number of block inventory slots. Default - 0 (no inventory). ### *size* Array of three integers. Default value is `[1, 1, 1]`. + +## Block fields + +Block fields allow you to write more data unique to a specified voxel than the user bits allow. + +Block fields are declared in the following format: + +```json +"fields": { + "name": {"type": "data_type"}, + "array_name": {"type": "data_type", "length": "array_length"} +} +``` + +In addition to `type` and `length`, the `convert-strategy` parameter determines the value conversion strategy when narrowing the data type. + +The parameter takes one of two values: +- `reset` - a value that does not exists in the new range will be reset to 0 +- `clamp` - the value will be reduced to the closest one in the new range + +Example: the number 231 when changing the field type from int16 to int8: +- in `reset` mode will turn into 0 +- in `clamp` mode will turn into 127 + +Available data types: + +| Type | Size | Description | +| ------- | --------- | ---------------------- | +| int8 | 1 byte | signed integer 8 bits | +| int16 | 2 bytes | signed integer 16 bits | +| int32 | 4 bytes | signed integer 32 bits | +| int64 | 8 bytes | integer signed 64 bits | +| float32 | 4 bytes | floating-point 32 bits | +| float64 | 8 bytes | floating-point 64 bits | +| char | 1 byte | character | + +- Currently, the total sum of the field sizes cannot exceed 240 bytes. +- A field without an array length specification is equivalent to an array of 1 element. +- A character array can be used to store UTF-8 strings. diff --git a/doc/en/scripting/builtins/libblock.md b/doc/en/scripting/builtins/libblock.md index 87176e20..fdb4fa4d 100644 --- a/doc/en/scripting/builtins/libblock.md +++ b/doc/en/scripting/builtins/libblock.md @@ -132,3 +132,29 @@ To use filter `dest` argument must be filled with some value(can be nil), it's d The function returns a table with the results or nil if the ray does not hit any block. The result will use the destination table instead of creating a new one if the optional argument specified. + +## Data fields + +```lua +-- writes a value to the specified block field +-- * throws an exception if the types are incompatible +-- * throws an exception when array is out of bounds +-- * does nothing if the block does not have the field +block.set_field( + x: int, y: int, z: int, + name: str, + value: bool|int|number|string, + [optional] index: int = 0 +) + +-- returns the value written to the block field +-- * returns nil if: +-- 1. the field does not exist +-- 2. no writes were made to any block field +-- * throws an exception when array is out of bounds +block.get_field( + x: int, y: int, z: int, + name: str, + [optional] index: int = 0 +) -> the stored value or nil +``` diff --git a/doc/ru/block-properties.md b/doc/ru/block-properties.md index 7e3a9768..8b2055e4 100644 --- a/doc/ru/block-properties.md +++ b/doc/ru/block-properties.md @@ -145,3 +145,44 @@ ### Размер блока - *size* Массив из трех целых чисел. Значение по-умолчанию - `[1, 1, 1]`. + +## Поля блока + +Поля блоков позволяет записывать больше уникальных для конкретного блока данных, чем это позволяют пользовательские биты. + +Поля блока объявляются в следующем формате: + +```json +"fields": { + "имя": {"type": "тип_данных"}, + "имя_массива": {"type": "тип_данных", "length": "длина_массива"} +} +``` + +Кроме `type` и `length` доступен параметр `convert-strategy` определяющий +стратегию конвертации значения при сужении типа данных. + +Параметр принимает одно из двух значений: +- `reset` - значение, не попадающее в новый диапазон, будет сброшено до 0 +- `clamp` - значение будет сведено к ближайшему в новом диапазоне + +Пример: число 231 при изменении типа поля с int16 до int8: +- в режиме `reset` превратится в 0 +- в режиме `clamp` превратится в 127 + + +Доступные типы данных: + +| Тип | Размер | Описание | +| ------- | -------- | ----------------------------- | +| int8 | 1 байт | целочисленный знаковый 8 бит | +| int16 | 2 байта | целочисленный знаковый 16 бит | +| int32 | 4 байта | целочисленный знаковый 32 бит | +| int64 | 8 байт | целочисленный знаковый 64 бит | +| float32 | 4 байта | вещественный 32 бит | +| float64 | 8 байт | вещественный 64 бит | +| char | 1 байт | символьный | + +- На данный момент общая сумма размеров полей не может превышать 240 байт. +- Поле без указания длины массива эквивалентно массиву из 1 элемента. +- Массив символьного типа может использоваться для хранения UTF-8 строк. diff --git a/doc/ru/scripting/builtins/libblock.md b/doc/ru/scripting/builtins/libblock.md index 150e776e..e7746a37 100644 --- a/doc/ru/scripting/builtins/libblock.md +++ b/doc/ru/scripting/builtins/libblock.md @@ -159,3 +159,29 @@ block.get_model(id: int) -> str -- возвращает массив из 6 текстур, назначенных на стороны блока block.get_textures(id: int) -> таблица строк ``` + +## Поля данных + +```lua +-- записывает значение в указанное поле блока +-- * бросает исключение при несовместимости типов +-- * бросает исключение при выходе за границы массива +-- * ничего не делает при отсутствии поля у блока +block.set_field( + x: int, y: int, z: int, + name: str, + value: bool|int|number|string, + [опционально] index: int = 0 +) + +-- возвращает значение записанное в поле блока +-- * возвращает nil если: +-- 1. поле не существует +-- 2. ни в одно поле блока не было произведено записи +-- * бросает исключение при выходе за границы массива +block.get_field( + x: int, y: int, z: int, + name: str, + [опционально] index: int = 0 +) -> хранимое значение или nil +``` diff --git a/src/coders/binary_json_spec.md b/doc/specs/binary_json_spec.md similarity index 92% rename from src/coders/binary_json_spec.md rename to doc/specs/binary_json_spec.md index e7b6f450..32158dd5 100644 --- a/src/coders/binary_json_spec.md +++ b/doc/specs/binary_json_spec.md @@ -49,9 +49,3 @@ int32 = 4byte int16 = 2byte byte = %x00-FF ``` - -## VoxelEngine format support - -Current implementation does not support types: bytes array, null, compressed document. - -All unsupported types will be implemented in future. diff --git a/doc/specs/outdated/region_file_spec_v2.md b/doc/specs/outdated/region_file_spec_v2.md new file mode 100644 index 00000000..4e4d2890 --- /dev/null +++ b/doc/specs/outdated/region_file_spec_v2.md @@ -0,0 +1,41 @@ +# Region File (version 2) + +File format BNF (RFC 5234): + +```bnf +file = header (*chunk) offsets complete file +header = magic %x02 %x00 magic number, version and reserved + zero byte + +magic = %x2E %x56 %x4F %x58 '.VOXREG\0' + %x52 %x45 %x47 %x00 + +chunk = int32 (*byte) byte array with size prefix +offsets = (1024*int32) offsets table +int32 = 4byte signed big-endian 32 bit integer +byte = %x00-FF 8 bit unsigned integer +``` + +C struct visualization: + +```c +typedef unsigned char byte; + +struct file { + // 10 bytes + struct { + char magic[8] = ".VOXREG"; + byte version = 2; + byte reserved = 0; + } header; + + struct { + int32_t size; // byteorder: big-endian + byte* data; + } chunks[1024]; // file does not contain zero sizes for missing chunks + + int32_t offsets[1024]; // byteorder: big-endian +}; +``` + +Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size). diff --git a/doc/specs/outdated/region_voxels_chunk_spec_v1.md b/doc/specs/outdated/region_voxels_chunk_spec_v1.md new file mode 100644 index 00000000..7467944c --- /dev/null +++ b/doc/specs/outdated/region_voxels_chunk_spec_v1.md @@ -0,0 +1,26 @@ +# Voxels Chunk (version 1) + +Voxel regions layer chunk structure. + +Values are separated for extRLE8 compression efficiency. + +File format BNF (RFC 5234): + +```bnf +chunk = (65536*byte) block indices (most significant bytes) + (65536*byte) block indices (least significant bytes) + (65536*byte) block states (most significant bytes) + (65536*byte) block states (least significant bytes) + +byte = %x00-FF 8 bit unsigned integer +``` + +65536 is number of voxels per chunk (16\*256\*16) + +## Block state + +Block state is encoded in 16 bits: +- 0-2 bits (3) - block rotation index +- 3-5 bits (3) - segment block bits +- 6-7 bits (2) - reserved +- 8-15 bits (8) - user bits diff --git a/doc/specs/region_file_spec.md b/doc/specs/region_file_spec.md new file mode 100644 index 00000000..30601798 --- /dev/null +++ b/doc/specs/region_file_spec.md @@ -0,0 +1,50 @@ +# Region File (version 3) + +File format BNF (RFC 5234): + +```bnf +file = header (*chunk) offsets complete file +header = magic %x02 byte magic number, version and compression + method + +magic = %x2E %x56 %x4F %x58 '.VOXREG\0' + %x52 %x45 %x47 %x00 + +chunk = uint32 uint32 (*byte) byte array with size and source size + prefix where source size is + decompressed chunk data size + +offsets = (1024*uint32) offsets table +int32 = 4byte unsigned big-endian 32 bit integer +byte = %x00-FF 8 bit unsigned integer +``` + +C struct visualization: + +```c +typedef unsigned char byte; + +struct file { + // 10 bytes + struct { + char magic[8] = ".VOXREG"; + byte version = 3; + byte compression; + } header; + + struct { + uint32_t size; // byteorder: little-endian + uint32_t sourceSize; // byteorder: little-endian + byte* data; + } chunks[1024]; // file does not contain zero sizes for missing chunks + + uint32_t offsets[1024]; // byteorder: little-endian +}; +``` + +Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size). + +Available compression methods: +0. no compression +1. extRLE8 +2. extRLE16 diff --git a/doc/specs/region_voxels_chunk_spec.md b/doc/specs/region_voxels_chunk_spec.md new file mode 100644 index 00000000..d43208cb --- /dev/null +++ b/doc/specs/region_voxels_chunk_spec.md @@ -0,0 +1,23 @@ +# Voxels Chunk (version 2) + +IDs and states are separated for extRLE16 compression efficiency. + +File format BNF (RFC 5234): + +```bnf +chunk = (65536*uint16) block ids + (65536*uint16) block states + +uint16 = 2byte 16 bit little-endian unsigned integer +byte = %x00-FF 8 bit unsigned integer +``` + +65536 is number of voxels per chunk (16\*256\*16) + +## Block state + +Block state is encoded in 16 bits: +- 0-2 bits (3) - block rotation index +- 3-5 bits (3) - segment block bits +- 6-7 bits (2) - reserved +- 8-15 bits (8) - user bits diff --git a/res/texts/en_US.txt b/res/texts/en_US.txt index f36f12a7..de5d950a 100644 --- a/res/texts/en_US.txt +++ b/res/texts/en_US.txt @@ -1,6 +1,9 @@ # Menu menu.missing-content=Missing Content! world.convert-request=Content indices have changed! Convert world files? +world.upgrade-request=World format is outdated! Convert world files? +world.convert-with-loss=Convert world with data loss? +world.convert-block-layouts=Blocks fields have changes! Convert world files? pack.remove-confirm=Do you want to erase all pack(s) content from the world forever? error.pack-not-found=Could not to find pack error.dependency-not-found=Dependency pack is not found diff --git a/res/texts/ru_RU.txt b/res/texts/ru_RU.txt index f8abdd9b..fbae5756 100644 --- a/res/texts/ru_RU.txt +++ b/res/texts/ru_RU.txt @@ -46,6 +46,9 @@ world.generators.default=Обычный world.generators.flat=Плоский world.Create World=Создать Мир world.convert-request=Есть изменения в индексах! Конвертировать мир? +world.upgrade-request=Формат мира устарел! Конвертировать мир? +world.convert-with-loss=Конвертировать мир с потерями? +world.convert-block-layouts=Есть изменения в полях блоков! Конвертировать мир? world.delete-confirm=Удалить мир безвозвратно? # Настройки diff --git a/src/coders/byte_utils.cpp b/src/coders/byte_utils.cpp index 84e33a30..380c932c 100644 --- a/src/coders/byte_utils.cpp +++ b/src/coders/byte_utils.cpp @@ -4,6 +4,8 @@ #include #include +#include "util/data_io.hpp" + void ByteBuilder::put(ubyte b) { buffer.push_back(b); } @@ -30,28 +32,24 @@ void ByteBuilder::put(const ubyte* arr, size_t size) { } void ByteBuilder::putInt16(int16_t val) { - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int16_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int16_t)); } void ByteBuilder::putInt32(int32_t val) { - buffer.reserve(buffer.size() + 4); - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); - buffer.push_back(static_cast(val >> 16 & 255)); - buffer.push_back(static_cast(val >> 24 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int32_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int32_t)); } void ByteBuilder::putInt64(int64_t val) { - buffer.reserve(buffer.size() + 8); - buffer.push_back(static_cast(val >> 0 & 255)); - buffer.push_back(static_cast(val >> 8 & 255)); - buffer.push_back(static_cast(val >> 16 & 255)); - buffer.push_back(static_cast(val >> 24 & 255)); - buffer.push_back(static_cast(val >> 32 & 255)); - buffer.push_back(static_cast(val >> 40 & 255)); - buffer.push_back(static_cast(val >> 48 & 255)); - buffer.push_back(static_cast(val >> 56 & 255)); + size_t size = buffer.size(); + buffer.resize(buffer.size() + sizeof(int64_t)); + val = dataio::h2le(val); + std::memcpy(buffer.data()+size, &val, sizeof(int64_t)); } void ByteBuilder::putFloat32(float val) { @@ -71,27 +69,18 @@ void ByteBuilder::set(size_t position, ubyte val) { } void ByteBuilder::setInt16(size_t position, int16_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position] = val >> 8 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int16_t)); } void ByteBuilder::setInt32(size_t position, int32_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position++] = val >> 8 & 255; - buffer[position++] = val >> 16 & 255; - buffer[position] = val >> 24 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int32_t)); } void ByteBuilder::setInt64(size_t position, int64_t val) { - buffer[position++] = val >> 0 & 255; - buffer[position++] = val >> 8 & 255; - buffer[position++] = val >> 16 & 255; - buffer[position++] = val >> 24 & 255; - - buffer[position++] = val >> 32 & 255; - buffer[position++] = val >> 40 & 255; - buffer[position++] = val >> 48 & 255; - buffer[position] = val >> 56 & 255; + val = dataio::h2le(val); + std::memcpy(buffer.data()+position, &val, sizeof(int64_t)); } std::vector ByteBuilder::build() { @@ -111,7 +100,7 @@ void ByteReader::checkMagic(const char* data, size_t size) { throw std::runtime_error("invalid magic number"); } for (size_t i = 0; i < size; i++) { - if (this->data[pos + i] != (ubyte)data[i]) { + if (this->data[pos + i] != static_cast(data[i])) { throw std::runtime_error("invalid magic number"); } } @@ -133,38 +122,33 @@ ubyte ByteReader::peek() { } int16_t ByteReader::getInt16() { - if (pos + 2 > size) { + if (pos + sizeof(int16_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 2; - return (static_cast(data[pos - 1]) << 8) | - (static_cast(data[pos - 2])); + int16_t value; + std::memcpy(&value, data + pos, sizeof(int16_t)); + pos += sizeof(int16_t); + return dataio::le2h(value); } int32_t ByteReader::getInt32() { - if (pos + 4 > size) { + if (pos + sizeof(int32_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 4; - return (static_cast(data[pos - 1]) << 24) | - (static_cast(data[pos - 2]) << 16) | - (static_cast(data[pos - 3]) << 8) | - (static_cast(data[pos - 4])); + int32_t value; + std::memcpy(&value, data + pos, sizeof(int32_t)); + pos += sizeof(int32_t); + return dataio::le2h(value); } int64_t ByteReader::getInt64() { - if (pos + 8 > size) { + if (pos + sizeof(int64_t) > size) { throw std::runtime_error("buffer underflow"); } - pos += 8; - return (static_cast(data[pos - 1]) << 56) | - (static_cast(data[pos - 2]) << 48) | - (static_cast(data[pos - 3]) << 40) | - (static_cast(data[pos - 4]) << 32) | - (static_cast(data[pos - 5]) << 24) | - (static_cast(data[pos - 6]) << 16) | - (static_cast(data[pos - 7]) << 8) | - (static_cast(data[pos - 8])); + int64_t value; + std::memcpy(&value, data + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + return dataio::le2h(value); } float ByteReader::getFloat32() { @@ -183,7 +167,7 @@ double ByteReader::getFloat64() { const char* ByteReader::getCString() { const char* cstr = reinterpret_cast(data + pos); - pos += strlen(cstr) + 1; + pos += std::strlen(cstr) + 1; return cstr; } diff --git a/src/coders/byte_utils.hpp b/src/coders/byte_utils.hpp index 3aefa1f0..103327c5 100644 --- a/src/coders/byte_utils.hpp +++ b/src/coders/byte_utils.hpp @@ -5,28 +5,27 @@ #include "typedefs.hpp" -/* byteorder: little-endian */ class ByteBuilder { std::vector buffer; public: - /* Write one byte (8 bit unsigned integer) */ + /// @brief Write one byte (8 bit unsigned integer) void put(ubyte b); - /* Write c-string (bytes array terminated with '\00') */ + /// @brief Write c-string (bytes array terminated with '\00') void putCStr(const char* str); - /* Write signed 16 bit integer */ + /// @brief Write signed 16 bit little-endian integer void putInt16(int16_t val); - /* Write signed 32 bit integer */ + /// @brief Write signed 32 bit integer void putInt32(int32_t val); - /* Write signed 64 bit integer */ + /// @brief Write signed 64 bit integer void putInt64(int64_t val); - /* Write 32 bit floating-point number */ + /// @brief Write 32 bit floating-point number void putFloat32(float val); - /* Write 64 bit floating-point number */ + /// @brief Write 64 bit floating-point number void putFloat64(double val); - /* Write string (uint32 length + bytes) */ + /// @brief Write string (uint32 length + bytes) void put(const std::string& s); - /* Write sequence of bytes without any header */ + /// @brief Write sequence of bytes without any header void put(const ubyte* arr, size_t size); void set(size_t position, ubyte val); @@ -44,7 +43,6 @@ public: std::vector build(); }; -/// byteorder: little-endian class ByteReader { const ubyte* data; size_t size; @@ -58,11 +56,11 @@ public: ubyte get(); /// @brief Read one byte (unsigned 8 bit integer) without pointer move ubyte peek(); - /// @brief Read signed 16 bit integer + /// @brief Read signed 16 bit little-endian integer int16_t getInt16(); - /// @brief Read signed 32 bit integer + /// @brief Read signed 32 bit little-endian integer int32_t getInt32(); - /// @brief Read signed 64 bit integer + /// @brief Read signed 64 bit little-endian integer int64_t getInt64(); /// @brief Read 32 bit floating-point number float getFloat32(); diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp new file mode 100644 index 00000000..9f7b551e --- /dev/null +++ b/src/coders/compression.cpp @@ -0,0 +1,107 @@ +#include "compression.hpp" + +#include +#include +#include + +#include "rle.hpp" +#include "gzip.hpp" +#include "util/BufferPool.hpp" + +using namespace compression; + +static util::BufferPool buffer_pools[] { + {255}, + {UINT16_MAX}, + {UINT16_MAX * 8}, +}; + +static std::shared_ptr get_buffer(size_t minSize) { + for (auto& pool : buffer_pools) { + if (minSize <= pool.getBufferSize()) { + return pool.get(); + } + } + return nullptr; +} + +static auto compress_rle( + const ubyte* src, + size_t srclen, + size_t& len, + size_t(*encodefunc)(const ubyte*, size_t, ubyte*) +) { + auto buffer = get_buffer(srclen * 2); + auto bytes = buffer.get(); + std::unique_ptr uptr; + if (bytes == nullptr) { + uptr = std::make_unique(srclen * 2); + bytes = uptr.get(); + } + len = encodefunc(src, srclen, bytes); + if (uptr) { + return uptr; + } + auto data = std::make_unique(len); + std::memcpy(data.get(), bytes, len); + return data; +} + +std::unique_ptr compression::compress( + const ubyte* src, size_t srclen, size_t& len, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: + return compress_rle(src, srclen, len, extrle::encode); + case Method::EXTRLE16: + return compress_rle(src, srclen, len, extrle::encode16); + case Method::GZIP: { + auto buffer = gzip::compress(src, srclen); + auto data = std::make_unique(buffer.size()); + std::memcpy(data.get(), buffer.data(), buffer.size()); + len = buffer.size(); + return data; + } + default: + throw std::runtime_error("not implemented"); + } +} + +std::unique_ptr compression::decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: { + auto decompressed = std::make_unique(dstlen); + extrle::decode(src, srclen, decompressed.get()); + return decompressed; + } + case Method::EXTRLE16: { + auto decompressed = std::make_unique(dstlen); + size_t decoded = extrle::decode16(src, srclen, decompressed.get()); + if (decoded != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(decoded)); + } + return decompressed; + } + case Method::GZIP: { + auto buffer = gzip::decompress(src, srclen); + if (buffer.size() != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(buffer.size())); + } + auto decompressed = std::make_unique(buffer.size()); + std::memcpy(decompressed.get(), buffer.data(), buffer.size()); + return decompressed; + } + default: + throw std::runtime_error("not implemented"); + } +} diff --git a/src/coders/compression.hpp b/src/coders/compression.hpp new file mode 100644 index 00000000..98b7a0e3 --- /dev/null +++ b/src/coders/compression.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "typedefs.hpp" + +namespace compression { + enum class Method { + NONE, EXTRLE8, EXTRLE16, GZIP + }; + + /// @brief Compress buffer + /// @param src source buffer + /// @param srclen length of the source buffer + /// @param len (out argument) length of result buffer + /// @param method compression method + /// @return compressed bytes array + /// @throws std::invalid_argument if compression method is NONE + std::unique_ptr compress( + const ubyte* src, size_t srclen, size_t& len, Method method + ); + + /// @brief Decompress buffer + /// @param src compressed buffer + /// @param srclen length of compressed buffer + /// @param dstlen max expected length of source buffer + /// @return decompressed bytes array + std::unique_ptr decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method); +} diff --git a/src/coders/rle.cpp b/src/coders/rle.cpp index 771d1a41..e99006e5 100644 --- a/src/coders/rle.cpp +++ b/src/coders/rle.cpp @@ -1,5 +1,7 @@ #include "rle.hpp" +#include "util/data_io.hpp" + size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) { size_t offset = 0; for (size_t i = 0; i < srclen;) { @@ -35,13 +37,52 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) { return offset; } +size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) { + auto src16 = reinterpret_cast(src); + auto dst16 = reinterpret_cast(dst); + size_t offset = 0; + for (size_t i = 0; i < srclen / 2;) { + uint16_t len = dataio::le2h(src16[i++]); + uint16_t c = dataio::le2h(src16[i++]); + for (size_t j = 0; j <= len; j++) { + dst16[offset++] = c; + } + } + return offset * 2; +} + +size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) { + if (srclen == 0) { + return 0; + } + auto src16 = reinterpret_cast(src); + auto dst16 = reinterpret_cast(dst); + size_t offset = 0; + uint16_t counter = 0; + uint16_t c = src16[0]; + for (size_t i = 1; i < srclen / 2; i++) { + uint16_t cnext = src16[i]; + if (cnext != c || counter == 0xFFFF) { + dst16[offset++] = dataio::h2le(counter); + dst16[offset++] = dataio::h2le(c); + c = cnext; + counter = 0; + } else { + counter++; + } + } + dst16[offset++] = dataio::h2le(counter); + dst16[offset++] = dataio::h2le(c); + return offset * 2; +} + size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) { size_t offset = 0; for (size_t i = 0; i < srclen;) { uint len = src[i++]; if (len & 0x80) { len &= 0x7F; - len |= ((uint)src[i++]) << 7; + len |= (static_cast(src[i++])) << 7; } ubyte c = src[i++]; for (size_t j = 0; j <= len; j++) { @@ -83,3 +124,70 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) { dst[offset++] = c; return offset; } + +size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) { + auto dst = reinterpret_cast(dst8); + size_t offset = 0; + for (size_t i = 0; i < srclen;) { + uint len = src[i++]; + bool widechar = len & 0x40; + if (len & 0x80) { + len &= 0x3F; + len |= (static_cast(src[i++])) << 6; + } else { + len &= 0x3F; + } + uint16_t c = src[i++]; + if (widechar) { + c |= ((static_cast(src[i++])) << 8); + } + for (size_t j = 0; j <= len; j++) { + dst[offset++] = c; + } + } + return offset * 2; +} + +size_t extrle::encode16(const ubyte* src8, size_t srclen, ubyte* dst) { + if (srclen == 0) { + return 0; + } + auto src = reinterpret_cast(src8); + size_t offset = 0; + uint counter = 0; + uint16_t c = src[0]; + for (size_t i = 1; i < srclen/2; i++) { + uint16_t cnext = src[i]; + if (cnext != c || counter == max_sequence16) { + if (counter >= 0x40) { + dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F); + dst[offset++] = counter >> 6; + } else { + dst[offset++] = counter | ((c > 255) << 6); + } + if (c > 255) { + dst[offset++] = c & 0xFF; + dst[offset++] = c >> 8; + } else { + dst[offset++] = c; + } + c = cnext; + counter = 0; + } else { + counter++; + } + } + if (counter >= 0x40) { + dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F); + dst[offset++] = counter >> 6; + } else { + dst[offset++] = counter | ((c > 255) << 6); + } + if (c > 255) { + dst[offset++] = c & 0xFF; + dst[offset++] = c >> 8; + } else { + dst[offset++] = c; + } + return offset; +} diff --git a/src/coders/rle.hpp b/src/coders/rle.hpp index 0692cc9a..c087ac69 100644 --- a/src/coders/rle.hpp +++ b/src/coders/rle.hpp @@ -5,10 +5,17 @@ namespace rle { size_t encode(const ubyte* src, size_t length, ubyte* dst); size_t decode(const ubyte* src, size_t length, ubyte* dst); + + size_t encode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst); } namespace extrle { constexpr uint max_sequence = 0x7FFF; size_t encode(const ubyte* src, size_t length, ubyte* dst); size_t decode(const ubyte* src, size_t length, ubyte* dst); + + constexpr uint max_sequence16 = 0x3FFF; + size_t encode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst); } diff --git a/src/constants.hpp b/src/constants.hpp index 15b3e732..da4acfc9 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -16,6 +16,12 @@ inline constexpr bool ENGINE_DEBUG_BUILD = true; inline const std::string ENGINE_VERSION_STRING = "0.23"; +/// @brief world regions format version +inline constexpr uint REGION_FORMAT_VERSION = 3; + +/// @brief max simultaneously open world region files +inline constexpr uint MAX_OPEN_REGION_FILES = 32; + inline constexpr blockid_t BLOCK_AIR = 0; inline constexpr itemid_t ITEM_EMPTY = 0; inline constexpr entityid_t ENTITY_NONE = 0; @@ -40,6 +46,7 @@ inline constexpr itemid_t ITEM_VOID = std::numeric_limits::max(); /// @brief max number of block definitions possible inline constexpr blockid_t MAX_BLOCKS = BLOCK_VOID; +/// @brief calculates a 1D array index from 3D array indices inline constexpr uint vox_index(uint x, uint y, uint z, uint w=CHUNK_W, uint d=CHUNK_D) { return (y * d + z) * w + x; } diff --git a/src/content/Content.hpp b/src/content/Content.hpp index 822cc418..375d0e25 100644 --- a/src/content/Content.hpp +++ b/src/content/Content.hpp @@ -24,15 +24,15 @@ namespace rigging { class SkeletonConfig; } -constexpr const char* contenttype_name(contenttype type) { +constexpr const char* contenttype_name(ContentType type) { switch (type) { - case contenttype::none: + case ContentType::NONE: return "none"; - case contenttype::block: + case ContentType::BLOCK: return "block"; - case contenttype::item: + case ContentType::ITEM: return "item"; - case contenttype::entity: + case ContentType::ENTITY: return "entity"; default: return "unknown"; @@ -40,13 +40,13 @@ constexpr const char* contenttype_name(contenttype type) { } class namereuse_error : public std::runtime_error { - contenttype type; + ContentType type; public: - namereuse_error(const std::string& msg, contenttype type) + namereuse_error(const std::string& msg, ContentType type) : std::runtime_error(msg), type(type) { } - inline contenttype getType() const { + inline ContentType getType() const { return type; } }; diff --git a/src/content/ContentBuilder.hpp b/src/content/ContentBuilder.hpp index d3e8f536..7961d1bf 100644 --- a/src/content/ContentBuilder.hpp +++ b/src/content/ContentBuilder.hpp @@ -12,8 +12,8 @@ template class ContentUnitBuilder { - std::unordered_map& allNames; - contenttype type; + std::unordered_map& allNames; + ContentType type; void checkIdentifier(const std::string& id) { const auto& found = allNames.find(id); @@ -28,7 +28,7 @@ public: std::vector names; ContentUnitBuilder( - std::unordered_map& allNames, contenttype type + std::unordered_map& allNames, ContentType type ) : allNames(allNames), type(type) { } @@ -62,11 +62,11 @@ class ContentBuilder { UptrsMap blockMaterials; UptrsMap skeletons; UptrsMap packs; - std::unordered_map allNames; + std::unordered_map allNames; public: - ContentUnitBuilder blocks {allNames, contenttype::block}; - ContentUnitBuilder items {allNames, contenttype::item}; - ContentUnitBuilder entities {allNames, contenttype::entity}; + ContentUnitBuilder blocks {allNames, ContentType::BLOCK}; + ContentUnitBuilder items {allNames, ContentType::ITEM}; + ContentUnitBuilder entities {allNames, ContentType::ENTITY}; ResourceIndicesSet resourceIndices {}; ~ContentBuilder(); diff --git a/src/content/ContentLUT.cpp b/src/content/ContentLUT.cpp deleted file mode 100644 index 28148e1c..00000000 --- a/src/content/ContentLUT.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "ContentLUT.hpp" - -#include - -#include "coders/json.hpp" -#include "constants.hpp" -#include "files/files.hpp" -#include "items/ItemDef.hpp" -#include "voxels/Block.hpp" -#include "world/World.hpp" -#include "files/WorldFiles.hpp" -#include "Content.hpp" - -ContentLUT::ContentLUT( - const ContentIndices* indices, size_t blocksCount, size_t itemsCount -) - : blocks(blocksCount, indices->blocks, BLOCK_VOID, contenttype::block), - items(itemsCount, indices->items, ITEM_VOID, contenttype::item) { -} - -template -static constexpr size_t get_entries_count( - const ContentUnitIndices& indices, const dv::value& list -) { - return list != nullptr - ? std::max(list.size(), indices.count()) - : indices.count(); -} - -std::shared_ptr ContentLUT::create( - const std::shared_ptr& worldFiles, - const fs::path& filename, - const Content* content -) { - auto worldInfo = worldFiles->readWorldInfo(); - if (!worldInfo.has_value()) { - return nullptr; - } - - auto root = files::read_json(filename); - auto& blocklist = root["blocks"]; - auto& itemlist = root["items"]; - - auto* indices = content->getIndices(); - size_t blocks_c = get_entries_count(indices->blocks, blocklist); - size_t items_c = get_entries_count(indices->items, itemlist); - - auto lut = std::make_shared(indices, blocks_c, items_c); - - lut->blocks.setup(blocklist, content->blocks); - lut->items.setup(itemlist, content->items); - - if (lut->hasContentReorder() || lut->hasMissingContent()) { - return lut; - } else { - return nullptr; - } -} - -std::vector ContentLUT::getMissingContent() const { - std::vector entries; - blocks.getMissingContent(entries); - items.getMissingContent(entries); - return entries; -} diff --git a/src/content/ContentLoader.cpp b/src/content/ContentLoader.cpp index 5c10b6e3..5fe39b1b 100644 --- a/src/content/ContentLoader.cpp +++ b/src/content/ContentLoader.cpp @@ -21,8 +21,10 @@ #include "util/stringutil.hpp" #include "voxels/Block.hpp" #include "data/dv_util.hpp" +#include "data/StructLayout.hpp" namespace fs = std::filesystem; +using namespace data; static debug::Logger logger("content-loader"); @@ -116,6 +118,30 @@ void ContentLoader::fixPackIndices() { } } +static void perform_user_block_fields( + const std::string& blockName, StructLayout& layout +) { + if (layout.size() > MAX_USER_BLOCK_FIELDS_SIZE) { + throw std::runtime_error( + util::quote(blockName) + + " fields total size exceeds limit (" + + std::to_string(layout.size()) + "/" + + std::to_string(MAX_USER_BLOCK_FIELDS_SIZE) + ")"); + } + for (const auto& field : layout) { + if (field.name.at(0) == '.') { + throw std::runtime_error( + util::quote(blockName) + " field " + field.name + + ": user field may not start with '.'"); + } + } + + std::vector fields; + fields.insert(fields.end(), layout.begin(), layout.end()); + // add built-in fields here + layout = StructLayout::create(fields); +} + void ContentLoader::loadBlock( Block& def, const std::string& name, const fs::path& file ) { @@ -250,6 +276,14 @@ void ContentLoader::loadBlock( root.at("ui-layout").get(def.uiLayout); root.at("inventory-size").get(def.inventorySize); root.at("tick-interval").get(def.tickInterval); + + if (root.has("fields")) { + def.dataStruct = std::make_unique(); + def.dataStruct->deserialize(root["fields"]); + + perform_user_block_fields(def.name, *def.dataStruct); + } + if (def.tickInterval == 0) { def.tickInterval = 1; } diff --git a/src/content/ContentReport.cpp b/src/content/ContentReport.cpp new file mode 100644 index 00000000..605b822c --- /dev/null +++ b/src/content/ContentReport.cpp @@ -0,0 +1,146 @@ +#include "ContentReport.hpp" + +#include + +#include "coders/json.hpp" +#include "constants.hpp" +#include "files/files.hpp" +#include "items/ItemDef.hpp" +#include "voxels/Block.hpp" +#include "world/World.hpp" +#include "files/WorldFiles.hpp" +#include "Content.hpp" + +ContentReport::ContentReport( + const ContentIndices* indices, + size_t blocksCount, + size_t itemsCount, + uint regionsVersion +) + : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK), + items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM), + regionsVersion(regionsVersion) { +} + +template +static constexpr size_t get_entries_count( + const ContentUnitIndices& indices, const dv::value& list +) { + return list != nullptr ? std::max(list.size(), indices.count()) + : indices.count(); +} + +static void process_blocks_data( + const Content* content, ContentReport& report, const dv::value& root +) { + for (const auto& [name, map] : root.asObject()) { + data::StructLayout layout; + layout.deserialize(map); + auto def = content->blocks.find(name); + if (def == nullptr) { + continue; + } + if (def->dataStruct == nullptr) { + ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE}; + report.issues.push_back(issue); + report.dataLoss.push_back(name + ": discard data"); + continue; + } + if (layout != *def->dataStruct) { + ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE}; + report.issues.push_back(issue); + report.dataLayoutsUpdated = true; + } + + auto incapatibility = layout.checkCompatibility(*def->dataStruct); + if (!incapatibility.empty()) { + for (const auto& error : incapatibility) { + report.dataLoss.push_back( + "[" + name + "] field " + error.name + " - " + + data::to_string(error.type) + ); + } + } + report.blocksDataLayouts[name] = std::move(layout); + } +} + +std::shared_ptr ContentReport::create( + const std::shared_ptr& worldFiles, + const fs::path& filename, + const Content* content +) { + auto worldInfo = worldFiles->readWorldInfo(); + if (!worldInfo.has_value()) { + return nullptr; + } + + auto root = files::read_json(filename); + // TODO: remove default value 2 in 0.24 + uint regionsVersion = 2U; + root.at("region-version").get(regionsVersion); + auto& blocklist = root["blocks"]; + auto& itemlist = root["items"]; + + auto* indices = content->getIndices(); + size_t blocks_c = get_entries_count(indices->blocks, blocklist); + size_t items_c = get_entries_count(indices->items, itemlist); + + auto report = std::make_shared( + indices, blocks_c, items_c, regionsVersion + ); + report->blocks.setup(blocklist, content->blocks); + report->items.setup(itemlist, content->items); + + if (root.has("blocks-data")) { + process_blocks_data(content, *report, root["blocks-data"]); + } + + report->buildIssues(); + + if (report->isUpgradeRequired() || report->hasContentReorder() || + report->hasMissingContent() || report->hasUpdatedLayouts()) { + return report; + } else { + return nullptr; + } +} + +template +static void build_issues( + std::vector& issues, const ContentUnitLUT& report +) { + auto type = report.getContentType(); + if (report.hasContentReorder()) { + issues.push_back(ContentIssue {ContentIssueType::REORDER, type}); + } + if (report.hasMissingContent()) { + issues.push_back(ContentIssue {ContentIssueType::MISSING, type}); + } +} + +void ContentReport::buildIssues() { + build_issues(issues, blocks); + build_issues(issues, items); + + if (regionsVersion < REGION_FORMAT_VERSION) { + for (int layer = REGION_LAYER_VOXELS; + layer < REGION_LAYERS_COUNT; + layer++) { + ContentIssue issue {ContentIssueType::REGION_FORMAT_UPDATE}; + issue.regionLayer = static_cast(layer); + issues.push_back(issue); + } + } +} + +const std::vector& ContentReport::getIssues() const { + return issues; +} + +std::vector ContentReport::getMissingContent() const { + std::vector entries; + blocks.getMissingContent(entries); + items.getMissingContent(entries); + return entries; +} diff --git a/src/content/ContentLUT.hpp b/src/content/ContentReport.hpp similarity index 58% rename from src/content/ContentLUT.hpp rename to src/content/ContentReport.hpp index 77531ba5..3476b872 100644 --- a/src/content/ContentLUT.hpp +++ b/src/content/ContentReport.hpp @@ -4,35 +4,57 @@ #include #include #include +#include #include "constants.hpp" #include "data/dv.hpp" #include "typedefs.hpp" #include "Content.hpp" +#include "data/StructLayout.hpp" +#include "files/world_regions_fwd.hpp" namespace fs = std::filesystem; -struct contententry { - contenttype type; +enum class ContentIssueType { + REORDER, + MISSING, + REGION_FORMAT_UPDATE, + BLOCK_DATA_LAYOUTS_UPDATE, +}; + +struct ContentIssue { + ContentIssueType issueType; + union { + ContentType contentType; + RegionLayerIndex regionLayer; + }; +}; + +struct ContentEntry { + ContentType type; std::string name; }; class WorldFiles; +/// @brief Content unit lookup table +/// @tparam T index type +/// @tparam U unit class template class ContentUnitLUT { std::vector indices; std::vector names; bool missingContent = false; bool reorderContent = false; + /// @brief index that will be used to mark missing unit T missingValue; - contenttype type; + ContentType type; public: ContentUnitLUT( size_t count, const ContentUnitIndices& unitIndices, T missingValue, - contenttype type + ContentType type ) : missingValue(missingValue), type(type) { for (size_t i = 0; i < count; i++) { @@ -57,11 +79,11 @@ public: } } } - void getMissingContent(std::vector& entries) const { + void getMissingContent(std::vector& entries) const { for (size_t i = 0; i < count(); i++) { if (indices[i] == missingValue) { auto& name = names[i]; - entries.push_back(contententry {type, name}); + entries.push_back(ContentEntry {type, name}); } } } @@ -80,6 +102,9 @@ public: reorderContent = true; } } + inline ContentType getContentType() const { + return type; + } inline size_t count() const { return indices.size(); } @@ -91,28 +116,54 @@ public: } }; -/// @brief Content indices lookup table or report -/// used to convert world with different indices +/// @brief Content incapatibility report used to convert world. /// Building with indices.json -class ContentLUT { +class ContentReport { public: ContentUnitLUT blocks; ContentUnitLUT items; + uint regionsVersion; - ContentLUT(const ContentIndices* indices, size_t blocks, size_t items); + std::unordered_map blocksDataLayouts; + std::vector issues; + std::vector dataLoss; - static std::shared_ptr create( + bool dataLayoutsUpdated = false; + + ContentReport( + const ContentIndices* indices, + size_t blocks, + size_t items, + uint regionsVersion + ); + + static std::shared_ptr create( const std::shared_ptr& worldFiles, const fs::path& filename, const Content* content ); + inline const std::vector& getDataLoss() const { + return dataLoss; + } + inline bool hasUpdatedLayouts() { + return dataLayoutsUpdated; + } + inline bool hasContentReorder() const { return blocks.hasContentReorder() || items.hasContentReorder(); } inline bool hasMissingContent() const { return blocks.hasMissingContent() || items.hasMissingContent(); } + inline bool isUpgradeRequired() const { + return regionsVersion < REGION_FORMAT_VERSION; + } + inline bool hasDataLoss() const { + return !dataLoss.empty(); + } + void buildIssues(); - std::vector getMissingContent() const; + const std::vector& getIssues() const; + std::vector getMissingContent() const; }; diff --git a/src/content/content_fwd.hpp b/src/content/content_fwd.hpp index 719fbfca..b1618593 100644 --- a/src/content/content_fwd.hpp +++ b/src/content/content_fwd.hpp @@ -5,7 +5,7 @@ class Content; class ContentPackRuntime; -enum class contenttype { none, block, item, entity }; +enum class ContentType { NONE, BLOCK, ITEM, ENTITY }; enum class ResourceType : size_t { CAMERA, LAST = CAMERA }; diff --git a/src/data/StructLayout.cpp b/src/data/StructLayout.cpp new file mode 100644 index 00000000..a3e0fd23 --- /dev/null +++ b/src/data/StructLayout.cpp @@ -0,0 +1,391 @@ +#include "StructLayout.hpp" + +#include +#include +#include +#include +#include + +#include "util/data_io.hpp" +#include "util/stringutil.hpp" + +using namespace data; + +static_assert(sizeof(float) == sizeof(int32_t)); +static_assert(sizeof(double) == sizeof(int64_t)); + +FieldType data::FieldType_from_string(std::string_view name) { + std::map map { + {"int8", FieldType::I8}, + {"int16", FieldType::I16}, + {"int32", FieldType::I32}, + {"int64", FieldType::I64}, + {"float32", FieldType::F32}, + {"float64", FieldType::F64}, + {"char", FieldType::CHAR}, + }; + return map.at(name); +} + +FieldConvertStrategy data::FieldConvertStrategy_from_string(std::string_view name) { + std::map map { + {"reset", FieldConvertStrategy::RESET}, + {"clamp", FieldConvertStrategy::CLAMP} + }; + return map.at(name); +} + +StructLayout StructLayout::create(const std::vector& fields) { + std::vector builtFields = fields; + std::unordered_map indices; + + for (Field& field : builtFields) { + field.size = sizeof_type(field.type) * field.elements; + } + std::sort(builtFields.begin(), builtFields.end(), + [](const Field& a, const Field& b) { + return a.size > b.size; + } + ); + int offset = 0; + for (int i = 0; i < builtFields.size(); i++) { + auto& field = builtFields[i]; + field.offset = offset; + indices[field.name] = i; + offset += field.size; + } + return StructLayout( + offset, std::move(builtFields), std::move(indices)); +} + +static inline constexpr bool is_integer_type(FieldType type) { + return (type >= FieldType::I8 && type <= FieldType::I64) || + type == FieldType::CHAR; +} + +static inline constexpr bool is_floating_point_type(FieldType type) { + return type == FieldType::F32 || type == FieldType::F64; +} + +static inline constexpr bool is_numeric_type(FieldType type) { + return is_floating_point_type(type) || is_integer_type(type); +} + +static inline FieldIncapatibilityType checkIncapatibility( + const Field& srcField, const Field& dstField +) { + auto type = FieldIncapatibilityType::NONE; + if (dstField.elements < srcField.elements) { + type = FieldIncapatibilityType::DATA_LOSS; + } + if (srcField.type == dstField.type) { + return type; + } + if (is_numeric_type(srcField.type) && is_numeric_type(dstField.type)) { + int sizediff = + sizeof_type(dstField.type) - sizeof_type(srcField.type); + if (sizediff < 0) { + type = std::max(type, FieldIncapatibilityType::DATA_LOSS); + } + } else { + type = std::max(type, FieldIncapatibilityType::TYPE_ERROR); + } + return type; +} + +static inline integer_t clamp_value(integer_t value, FieldType type) { + auto typesize = sizeof_type(type) * CHAR_BIT; + integer_t minval = -(1 << (typesize-1)); + integer_t maxval = (1 << (typesize-1))-1; + return std::min(maxval, std::max(minval, value)); +} + +static void reset_integer( + const StructLayout& srcLayout, + const StructLayout& dstLayout, + const Field& field, + const Field& dstField, + const ubyte* src, + ubyte* dst +) { + int elements = std::min(field.elements, dstField.elements); + for (int i = 0; i < elements; i++) { + auto value = srcLayout.getInteger(src, field.name, i); + auto clamped = clamp_value(value, dstField.type); + if (dstField.convertStrategy == FieldConvertStrategy::CLAMP) { + value = clamped; + } else { + if (clamped != value) { + value = 0; + } + } + dstLayout.setInteger(dst, value, field.name, i); + } +} + +static void reset_number( + const StructLayout& srcLayout, + const StructLayout& dstLayout, + const Field& field, + const Field& dstField, + const ubyte* src, + ubyte* dst +) { + int elements = std::min(field.elements, dstField.elements); + for (int i = 0; i < elements; i++) { + auto value = srcLayout.getNumber(src, field.name, i); + dstLayout.setNumber(dst, value, field.name, i); + } +} + +void StructLayout::convert( + const StructLayout& srcLayout, + const ubyte* src, + ubyte* dst, + bool allowDataLoss +) const { + std::memset(dst, 0, totalSize); + for (const Field& field : srcLayout.fields) { + auto dstField = getField(field.name); + if (dstField == nullptr) { + continue; + } + auto type = checkIncapatibility(field, *dstField); + if (type == FieldIncapatibilityType::TYPE_ERROR) { + continue; + } + // can't just memcpy, because field type may be changed without data loss + if (is_integer_type(field.type) || + (is_floating_point_type(field.type) && + is_integer_type(dstField->type))) { + reset_integer(srcLayout, *this, field, *dstField, src, dst); + } else if (is_floating_point_type(dstField->type)) { + reset_number(srcLayout, *this, field, *dstField, src, dst); + } + } +} + +std::vector StructLayout::checkCompatibility( + const StructLayout& dstLayout +) { + std::vector report; + for (const Field& field : fields) { + auto dstField = dstLayout.getField(field.name); + if (dstField == nullptr) { + report.push_back({field.name, FieldIncapatibilityType::MISSING}); + continue; + } + auto type = checkIncapatibility(field, *dstField); + if (type != FieldIncapatibilityType::NONE) { + report.push_back({field.name, type}); + } + } + return report; +} + +const Field& StructLayout::requireField(const std::string& name) const { + auto found = indices.find(name); + if (found == indices.end()) { + throw std::runtime_error("field '"+name+"' does not exist"); + } + return *&fields.at(found->second); +} + + +template +static void set_int(ubyte* dst, integer_t value) { + T out_value = static_cast(value); + out_value = dataio::le2h(out_value); + *reinterpret_cast(dst) = out_value; +} + +void StructLayout::setInteger( + ubyte* dst, integer_t value, const Field& field, int index +) const { + if (index < 0 || index >= field.elements) { + throw std::out_of_range( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = dst + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::I8: set_int(ptr, value); break; + case FieldType::I16: set_int(ptr, value); break; + case FieldType::I32: set_int(ptr, value); break; + case FieldType::I64: set_int(ptr, value); break; + case FieldType::CHAR: set_int(ptr, value); break; + case FieldType::F32: + case FieldType::F64: + setNumber(dst, static_cast(value), field, index); + break; + default: + throw std::runtime_error("type error"); + } +} + +void StructLayout::setNumber( + ubyte* dst, number_t value, const Field& field, int index +) const { + if (index < 0 || index >= field.elements) { + throw std::out_of_range( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = dst + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::F32: { + float fval = static_cast(value); + int32_t ival; + std::memcpy(&ival, &fval, sizeof(int32_t)); + set_int(ptr, ival); + break; + } + case FieldType::F64: { + double fval = static_cast(value); + int64_t ival; + std::memcpy(&ival, &fval, sizeof(int64_t)); + set_int(ptr, ival); + break; + } + default: + throw std::runtime_error("type error"); + } +} + +size_t StructLayout::setAscii( + ubyte* dst, std::string_view value, const std::string& name +) const { + const auto& field = requireField(name); + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto ptr = reinterpret_cast(dst + field.offset); + auto size = std::min(value.size(), static_cast(field.elements)); + std::memcpy(ptr, value.data(), size); + if (size < field.elements) { + std::memset(ptr + size, 0, field.elements - size); + } + return size; +} + +size_t StructLayout::setUnicode( + ubyte* dst, std::string_view value, const Field& field +) const { + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto text = std::string_view(value.data(), value.size()); + size_t size = util::crop_utf8(text, field.elements); + auto ptr = reinterpret_cast(dst + field.offset); + std::memcpy(ptr, value.data(), size); + if (size < field.elements) { + std::memset(ptr + size, 0, field.elements - size); + } + return size; +} + +template +static T get_int(const ubyte* src) { + return dataio::le2h(*reinterpret_cast(src)); +} + +integer_t StructLayout::getInteger( + const ubyte* src, const Field& field, int index +) const { + if (index < 0 || index >= field.elements) { + throw std::out_of_range( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = src + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::I8: return get_int(ptr); + case FieldType::I16: return get_int(ptr); + case FieldType::I32: return get_int(ptr); + case FieldType::I64: return get_int(ptr); + case FieldType::CHAR: return get_int(ptr); + default: + throw std::runtime_error("type error"); + } +} + +number_t StructLayout::getNumber( + const ubyte* src, const Field& field, int index +) const { + if (index < 0 || index >= field.elements) { + throw std::out_of_range( + "index out of bounds [0, "+std::to_string(field.elements)+"]"); + } + auto ptr = src + field.offset + index * sizeof_type(field.type); + switch (field.type) { + case FieldType::F32: { + float fval; + auto ival = get_int(ptr); + std::memcpy(&fval, &ival, sizeof(float)); + return fval; + } + case FieldType::F64: { + double fval; + auto ival = get_int(ptr); + std::memcpy(&fval, &ival, sizeof(double)); + return fval; + } + case FieldType::I8: + case FieldType::I16: + case FieldType::I32: + case FieldType::I64: + case FieldType::CHAR: + return getInteger(src, field, index); + default: + throw std::runtime_error("type error"); + } +} + +std::string_view StructLayout::getChars( + const ubyte* src, const Field& field +) const { + if (field.type != FieldType::CHAR) { + throw std::runtime_error("'char' field type required"); + } + auto ptr = reinterpret_cast(src + field.offset); + return std::string_view(ptr, strnlen(ptr, field.elements)); +} + +dv::value StructLayout::serialize() const { + auto map = dv::object(); + for (const auto& [name, index] : indices) { + auto& fieldmap = map.object(name); + const auto& field = fields[index]; + fieldmap["type"] = to_string(field.type); + if (field.elements != 1) { + fieldmap["length"] = field.elements; + } + if (field.convertStrategy != FieldConvertStrategy::RESET) { + fieldmap["convert-strategy"] = to_string(field.convertStrategy); + } + } + return map; +} + +void StructLayout::deserialize(const dv::value& src) { + std::vector fields; + for (const auto& [name, fieldmap] : src.asObject()) { + const auto& typeName = fieldmap["type"].asString(); + FieldType type = FieldType_from_string(typeName); + + int elements = 1; + fieldmap.at("length").get(elements); + if (elements <= 0) { + throw std::runtime_error( + "invalid field " + util::quote(name) + " length: " + + std::to_string(elements) + ); + } + + auto convertStrategy = FieldConvertStrategy::RESET; + if (fieldmap.has("convert-strategy")) { + convertStrategy = FieldConvertStrategy_from_string( + fieldmap["convert-strategy"].asString() + ); + } + fields.push_back(Field (type, name, elements, convertStrategy)); + } + *this = create(fields); +} diff --git a/src/data/StructLayout.hpp b/src/data/StructLayout.hpp new file mode 100644 index 00000000..02a5a435 --- /dev/null +++ b/src/data/StructLayout.hpp @@ -0,0 +1,280 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "typedefs.hpp" +#include "interfaces/Serializable.hpp" + +namespace data { + enum class FieldType { + I8=0, I16, I32, I64, F32, F64, CHAR + }; + + inline std::string to_string(FieldType type) { + const char* names[] = { + "int8", "int16", "int32", "int64", "float32", "float64", "char" + }; + return names[static_cast(type)]; + } + FieldType FieldType_from_string(std::string_view name); + + /// @brief Sorted by severity + enum class FieldIncapatibilityType { + NONE = 0, + DATA_LOSS, + TYPE_ERROR, + MISSING, + }; + inline const char* to_string(FieldIncapatibilityType type) { + const char* names[] = { + "none", "data_loss", "type_error", "missing" + }; + return names[static_cast(type)]; + } + + struct FieldIncapatibility { + std::string name; + FieldIncapatibilityType type; + }; + + inline constexpr int sizeof_type(FieldType type) { + const int sizes[] = { + 1, 2, 4, 8, 4, 8, 1 + }; + return sizes[static_cast(type)]; + } + + class dataloss_error : public std::runtime_error { + public: + dataloss_error(const std::string& message) : std::runtime_error(message) {} + }; + + /// @brief Strategy will be used if value can't be left the same on conversion + enum class FieldConvertStrategy { + /// @brief Reset field value + RESET = 0, + /// @brief Clamp field value if out of type range + CLAMP + }; + + inline const char* to_string(FieldConvertStrategy strategy) { + const char* names[] = { + "reset", "clamp" + }; + return names[static_cast(strategy)]; + } + FieldConvertStrategy FieldConvertStrategy_from_string(std::string_view name); + + struct Field { + FieldType type; + std::string name; + /// @brief Number of field elements (array size) + int elements; + /// @brief Strategy will be used in data loss case + FieldConvertStrategy convertStrategy; + /// @brief Byte offset of the field + int offset; + /// @brief Byte size of the field + int size; + + Field( + FieldType type, + std::string name, + int elements, + FieldConvertStrategy strategy=FieldConvertStrategy::RESET + ) : type(type), + name(std::move(name)), + elements(elements), + convertStrategy(strategy), + offset(0), + size(0) {} + + bool operator==(const Field& o) const { + return type == o.type && + name == o.name && + elements == o.elements && + convertStrategy == o.convertStrategy && + offset == o.offset && + size == o.size; + } + bool operator!=(const Field& o) const { + return !operator==(o); + } + }; + + class StructLayout : public Serializable { + int totalSize; + std::vector fields; + std::unordered_map indices; + + StructLayout( + int totalSize, + std::vector fields, + std::unordered_map indices + ) : totalSize(totalSize), + fields(std::move(fields)), + indices(std::move(indices)) + {} + public: + StructLayout() : StructLayout(0, {}, {}) {} + + bool operator==(const StructLayout& o) const { + // if fields are completely equal then structures are equal + return fields == o.fields; + } + bool operator!=(const StructLayout& o) const { + return !operator==(o); + } + + /// @brief Get field by name. Returns nullptr if field not found. + /// @param name field name + /// @return nullable field pointer + [[nodiscard]] + const Field* getField(const std::string& name) const { + auto found = indices.find(name); + if (found == indices.end()) { + return nullptr; + } + return &fields.at(found->second); + } + + /// @brief Get field by name + /// @throws std::runtime_exception - field not found + /// @param name field name + /// @return read-only field reference + const Field& requireField(const std::string& name) const; + + [[nodiscard]] + integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const { + return getInteger(src, requireField(name), index); + } + + /// @brief Get integer from specified field. + /// Types: (i8, i16, i32, i64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param src source buffer + /// @param field target field + /// @param index array index + /// @return field value + [[nodiscard]] + integer_t getInteger(const ubyte* src, const Field& field, int index=0) const; + + [[nodiscard]] + number_t getNumber(const ubyte* src, const std::string& name, int index=0) const { + return getNumber(src, requireField(name), index); + } + + /// @brief Get floating-point number from specified field. + /// Types: (f32, f64, i8, i16, i32, i64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param src source buffer + /// @param field target field + /// @param index array index + /// @return field value + [[nodiscard]] + number_t getNumber(const ubyte* src, const Field& field, int index=0) const; + + [[nodiscard]] + std::string_view getChars(const ubyte* src, const std::string& name) const { + return getChars(src, requireField(name)); + } + + /// @brief Get read-only chars array as string_view. + /// @param src source buffer + /// @param field target field + [[nodiscard]] + std::string_view getChars(const ubyte* src, const Field& field) const; + + void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0) const { + setInteger(dst, value, requireField(name), index); + } + + /// @brief Set field integer value. + /// Types: (i8, i16, i32, i64, f32, f64, char) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param dst destination buffer + /// @param value value + /// @param field target field + /// @param index array index + void setInteger(ubyte* dst, integer_t value, const Field& field, int index=0) const; + + void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0) const { + setNumber(dst, value, requireField(name), index); + } + + /// @brief Set field numeric value. + /// Types: (f32, f64) + /// @throws std::runtime_exception - field not found + /// @throws std::out_of_range - index is out of range [0, elements-1] + /// @param dst destination buffer + /// @param value value + /// @param field target field + /// @param index array index + void setNumber(ubyte* dst, number_t value, const Field& field, int index=0) const; + + /// @brief Replace chars array to given ASCII string + /// @throws std::runtime_exception - field not found + /// @see StructMapper::setUnicode - utf-8 version of setAscii + /// @param dst destination buffer + /// @param value ASCII string + /// @param name field name + /// @return number of written string chars + size_t setAscii(ubyte* dst, std::string_view value, const std::string& name) const; + + size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name) const { + return setUnicode(dst, value, requireField(name)); + } + + /// @brief Unicode-safe version of setAscii + /// @throws std::runtime_exception - field not found + /// @param dst destination buffer + /// @param value utf-8 string + /// @param name field name + /// @return number of written string chars + size_t setUnicode(ubyte* dst, std::string_view value, const Field& field) const; + + /// @return total structure size (bytes) + [[nodiscard]] size_t size() const { + return totalSize; + } + + [[nodiscard]] const auto begin() const { + return fields.begin(); + } + + [[nodiscard]] const auto end() const { + return fields.end(); + } + + /// @brief Convert structure data from srcLayout to this layout. + /// @param srcLayout source structure layout + /// @param src source data + /// @param dst destination buffer + /// (size must be enough to store converted structure) + /// @param allowDataLoss allow to drop fields that are not present in + /// this layout or have incompatible types + /// @throws data::dataloss_error - data loss detected and allowDataLoss + /// is set to false + void convert( + const StructLayout& srcLayout, + const ubyte* src, + ubyte* dst, + bool allowDataLoss) const; + + std::vector checkCompatibility( + const StructLayout& dstLayout); + + [[nodiscard]] + static StructLayout create(const std::vector& fields); + + dv::value serialize() const override; + void deserialize(const dv::value& src) override; + }; +} diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp new file mode 100644 index 00000000..027d6bf3 --- /dev/null +++ b/src/files/RegionsLayer.cpp @@ -0,0 +1,241 @@ +#include "WorldRegions.hpp" + +#include + +#include "util/data_io.hpp" + +#define REGION_FORMAT_MAGIC ".VOXREG" + +static fs::path get_region_filename(int x, int z) { + return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); +} + +/// @brief Read missing chunks data (null pointers) from region file +static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) { + auto* chunks = region->getChunks(); + auto sizes = region->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; + int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; + if (chunks[i] == nullptr) { + chunks[i] = RegionsLayer::readChunkData( + chunk_x, chunk_z, sizes[i][0], sizes[i][1], file); + } + } +} + +regfile::regfile(fs::path filename) : file(std::move(filename)) { + if (file.length() < REGION_HEADER_SIZE) + throw std::runtime_error("incomplete region file header"); + char header[REGION_HEADER_SIZE]; + file.read(header, REGION_HEADER_SIZE); + + // avoid of use strcmp_s + if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) != + REGION_FORMAT_MAGIC) { + throw std::runtime_error("invalid region file magic number"); + } + version = header[8]; + if (static_cast(version) > REGION_FORMAT_VERSION) { + throw illegal_region_format( + "region format " + std::to_string(version) + " is not supported" + ); + } +} + +std::unique_ptr regfile::read(int index, uint32_t& size, uint32_t& srcSize) { + size_t file_size = file.length(); + size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; + + uint32_t buff32; + file.seekg(table_offset + index * 4); + file.read(reinterpret_cast(&buff32), 4); + uint32_t offset = dataio::le2h(buff32); + if (offset == 0) { + return nullptr; + } + + file.seekg(offset); + file.read(reinterpret_cast(&buff32), 4); + size = dataio::le2h(buff32); + file.read(reinterpret_cast(&buff32), 4); + srcSize = dataio::le2h(buff32); + + auto data = std::make_unique(size); + file.read(reinterpret_cast(data.get()), size); + return data; +} + + +void RegionsLayer::closeRegFile(glm::ivec2 coord) { + openRegFiles.erase(coord); + regFilesCv.notify_one(); +} + +regfile_ptr RegionsLayer::useRegFile(glm::ivec2 coord) { + auto* file = openRegFiles[coord].get(); + file->inUse = true; + return regfile_ptr(file, ®FilesCv); +} + +// Marks regfile as used and unmarks when shared_ptr dies +regfile_ptr RegionsLayer::getRegFile(glm::ivec2 coord, bool create) { + { + std::lock_guard lock(regFilesMutex); + const auto found = openRegFiles.find(coord); + if (found != openRegFiles.end()) { + if (found->second->inUse) { + throw std::runtime_error("regfile is currently in use"); + } + return useRegFile(found->first); + } + } + if (create) { + return createRegFile(coord); + } + return nullptr; +} + +regfile_ptr RegionsLayer::createRegFile(glm::ivec2 coord) { + auto file = folder / get_region_filename(coord[0], coord[1]); + if (!fs::exists(file)) { + return nullptr; + } + if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { + std::unique_lock lock(regFilesMutex); + while (true) { + bool closed = false; + // FIXME: bad choosing algorithm + for (auto& entry : openRegFiles) { + if (!entry.second->inUse) { + closeRegFile(entry.first); + closed = true; + break; + } + } + if (closed) { + break; + } + // notified when any regfile gets out of use or closed + regFilesCv.wait(lock); + } + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } else { + std::lock_guard lock(regFilesMutex); + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } +} + +WorldRegion* RegionsLayer::getRegion(int x, int z) { + std::lock_guard lock(mapMutex); + auto found = regions.find({x, z}); + if (found == regions.end()) { + return nullptr; + } + return found->second.get(); +} + +fs::path RegionsLayer::getRegionFilePath(int x, int z) const { + return folder / get_region_filename(x, z); +} + +WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) { + if (auto region = getRegion(x, z)) { + return region; + } + std::lock_guard lock(mapMutex); + auto region_ptr = std::make_unique(); + auto region = region_ptr.get(); + regions[{x, z}] = std::move(region_ptr); + return region; +} + +ubyte* RegionsLayer::getData(int x, int z, uint32_t& size, uint32_t& srcSize) { + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + + WorldRegion* region = getOrCreateRegion(regionX, regionZ); + ubyte* data = region->getChunkData(localX, localZ); + if (data == nullptr) { + auto regfile = getRegFile({regionX, regionZ}); + if (regfile != nullptr) { + auto dataptr = readChunkData(x, z, size, srcSize, regfile.get()); + if (dataptr) { + data = dataptr.get(); + region->put(localX, localZ, std::move(dataptr), size, srcSize); + } + } + } + if (data != nullptr) { + auto sizevec = region->getChunkDataSize(localX, localZ); + size = sizevec[0]; + srcSize = sizevec[1]; + return data; + } + return nullptr; +} + +void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) { + fs::path filename = folder / get_region_filename(x, z); + + glm::ivec2 regcoord(x, z); + if (auto regfile = getRegFile(regcoord, false)) { + fetch_chunks(entry, x, z, regfile.get()); + + std::lock_guard lock(regFilesMutex); + regfile.reset(); + closeRegFile(regcoord); + } + + char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; + header[8] = REGION_FORMAT_VERSION; + header[9] = static_cast(compression); // FIXME + std::ofstream file(filename, std::ios::out | std::ios::binary); + file.write(header, REGION_HEADER_SIZE); + + size_t offset = REGION_HEADER_SIZE; + uint32_t intbuf; + uint offsets[REGION_CHUNKS_COUNT] {}; + + auto region = entry->getChunks(); + auto sizes = entry->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + ubyte* chunk = region[i].get(); + if (chunk == nullptr) { + continue; + } + offsets[i] = offset; + + auto sizevec = sizes[i]; + uint32_t compressedSize = sizevec[0]; + uint32_t srcSize = sizevec[1]; + + intbuf = dataio::h2le(compressedSize); + file.write(reinterpret_cast(&intbuf), 4); + offset += 4; + + intbuf = dataio::h2le(srcSize); + file.write(reinterpret_cast(&intbuf), 4); + offset += 4; + + file.write(reinterpret_cast(chunk), compressedSize); + offset += compressedSize; + } + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + intbuf = dataio::h2le(offsets[i]); + file.write(reinterpret_cast(&intbuf), 4); + } +} + +std::unique_ptr RegionsLayer::readChunkData( + int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile +) { + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + int chunkIndex = localZ * REGION_SIZE + localX; + return rfile->read(chunkIndex, size, srcSize); +} diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index 306b1a1a..46b85868 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -5,49 +5,140 @@ #include #include -#include "content/ContentLUT.hpp" +#include "content/ContentReport.hpp" +#include "files/compatibility.hpp" #include "debug/Logger.hpp" #include "files/files.hpp" #include "objects/Player.hpp" #include "util/ThreadPool.hpp" #include "voxels/Chunk.hpp" +#include "items/Inventory.hpp" +#include "voxels/Block.hpp" #include "WorldFiles.hpp" namespace fs = std::filesystem; static debug::Logger logger("world-converter"); -class ConverterWorker : public util::Worker { +class ConverterWorker : public util::Worker { std::shared_ptr converter; public: ConverterWorker(std::shared_ptr converter) : converter(std::move(converter)) { } - int operator()(const std::shared_ptr& task) override { + int operator()(const std::shared_ptr& task) override { converter->convert(*task); return 0; } }; +void WorldConverter::addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType +) { + const auto& regions = wfile->getRegions(); + auto regionsFolder = regions.getRegionsFolder(layerid); + if (!fs::is_directory(regionsFolder)) { + return; + } + for (const auto& file : fs::directory_iterator(regionsFolder)) { + int x, z; + std::string name = file.path().stem().string(); + if (!WorldRegions::parseRegionFilename(name, x, z)) { + logger.error() << "could not parse region name " << name; + continue; + } + tasks.push(ConvertTask {taskType, file.path(), x, z, layerid}); + } +} + +void WorldConverter::createUpgradeTasks() { + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) { + continue; + } + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION); + } +} + +void WorldConverter::createConvertTasks() { + auto handleReorder = [=](ContentType contentType) { + switch (contentType) { + case ContentType::BLOCK: + addRegionsTasks( + REGION_LAYER_VOXELS, + ConvertTaskType::VOXELS + ); + break; + case ContentType::ITEM: + addRegionsTasks( + REGION_LAYER_INVENTORIES, + ConvertTaskType::INVENTORIES + ); + break; + default: + break; + } + }; + + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + switch (issue.issueType) { + case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE: + case ContentIssueType::REGION_FORMAT_UPDATE: + break; + case ContentIssueType::MISSING: + throw std::runtime_error("issue can't be resolved"); + case ContentIssueType::REORDER: + handleReorder(issue.contentType); + break; + } + } + + tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()}); +} + +void WorldConverter::createBlockFieldsConvertTasks() { + // blocks data conversion requires correct block indices + // so it must be done AFTER voxels conversion + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + switch (issue.issueType) { + case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE: + addRegionsTasks( + REGION_LAYER_BLOCKS_DATA, + ConvertTaskType::CONVERT_BLOCKS_DATA + ); + break; + default: + break; + } + } +} + WorldConverter::WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr lut + std::shared_ptr reportPtr, + ConvertMode mode ) : wfile(worldFiles), - lut(std::move(lut)), - content(content) { - fs::path regionsFolder = - wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS); - if (!fs::is_directory(regionsFolder)) { - logger.error() << "nothing to convert"; - return; - } - tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()} - ); - for (const auto& file : fs::directory_iterator(regionsFolder)) { - tasks.push(convert_task {convert_task_type::region, file.path()}); + report(std::move(reportPtr)), + content(content), + mode(mode) +{ + switch (mode) { + case ConvertMode::UPGRADE: + createUpgradeTasks(); + break; + case ConvertMode::REINDEX: + createConvertTasks(); + break; + case ConvertMode::BLOCK_FIELDS: + createBlockFieldsConvertTasks(); + break; } } @@ -57,11 +148,13 @@ WorldConverter::~WorldConverter() { std::shared_ptr WorldConverter::startTask( const std::shared_ptr& worldFiles, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const runnable& onDone, + ConvertMode mode, bool multithreading ) { - auto converter = std::make_shared(worldFiles, content, lut); + auto converter = std::make_shared( + worldFiles, content, report, mode); if (!multithreading) { converter->setOnComplete([=]() { converter->write(); @@ -69,15 +162,15 @@ std::shared_ptr WorldConverter::startTask( }); return converter; } - auto pool = std::make_shared>( + auto pool = std::make_shared>( "converter-pool", [=]() { return std::make_shared(converter); }, [=](int&) {} ); auto& converterTasks = converter->tasks; while (!converterTasks.empty()) { - const convert_task& task = converterTasks.front(); - auto ptr = std::make_shared(task); + const ConvertTask& task = converterTasks.front(); + auto ptr = std::make_shared(task); pool->enqueueJob(ptr); converterTasks.pop(); } @@ -88,39 +181,85 @@ std::shared_ptr WorldConverter::startTask( return pool; } -void WorldConverter::convertRegion(const fs::path& file) const { - int x, z; - std::string name = file.stem().string(); - if (!WorldRegions::parseRegionFilename(name, x, z)) { - logger.error() << "could not parse name " << name; - return; - } - logger.info() << "converting region " << name; - wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) { - if (lut) { - Chunk::convert(data, lut.get()); - } - return true; +void WorldConverter::upgradeRegion( + const fs::path& file, int x, int z, RegionLayerIndex layer +) const { + auto path = wfile->getRegions().getRegionFilePath(layer, x, z); + auto bytes = files::read_bytes_buffer(path); + auto buffer = compatibility::convert_region_2to3(bytes, layer); + files::write_bytes(path, buffer.data(), buffer.size()); +} + +void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const { + logger.info() << "converting voxels region " << x << "_" << z; + wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, + [=](std::unique_ptr data, uint32_t*) { + Chunk::convert(data.get(), report.get()); + return data; + }); +} + +void WorldConverter::convertInventories(const fs::path& file, int x, int z) const { + logger.info() << "converting inventories region " << x << "_" << z; + wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) { + inventory->convert(report.get()); }); } void WorldConverter::convertPlayer(const fs::path& file) const { logger.info() << "converting player " << file.u8string(); auto map = files::read_json(file); - Player::convert(map, lut.get()); + Player::convert(map, report.get()); files::write_json(file, map); } -void WorldConverter::convert(const convert_task& task) const { +void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const { + logger.info() << "converting blocks data"; + wfile->getRegions().processBlocksData(x, z, + [=](BlocksMetadata* heap, std::unique_ptr voxelsData) { + Chunk chunk(0, 0); + chunk.decode(voxelsData.get()); + + const auto& indices = content->getIndices()->blocks; + + BlocksMetadata newHeap; + for (const auto& entry : *heap) { + size_t index = entry.index; + const auto& def = indices.require(chunk.voxels[index].id); + const auto& newStruct = *def.dataStruct; + const auto& found = report.blocksDataLayouts.find(def.name); + if (found == report.blocksDataLayouts.end()) { + logger.error() << "no previous fields layout found for block" + << def.name << " - discard"; + continue; + } + const auto& prevStruct = found->second; + uint8_t* dst = newHeap.allocate(index, newStruct.size()); + newStruct.convert(prevStruct, entry.data(), dst, true); + } + *heap = std::move(newHeap); + }); +} + +void WorldConverter::convert(const ConvertTask& task) const { if (!fs::is_regular_file(task.file)) return; switch (task.type) { - case convert_task_type::region: - convertRegion(task.file); + case ConvertTaskType::UPGRADE_REGION: + upgradeRegion(task.file, task.x, task.z, task.layer); break; - case convert_task_type::player: + case ConvertTaskType::VOXELS: + convertVoxels(task.file, task.x, task.z); + break; + case ConvertTaskType::INVENTORIES: + convertInventories(task.file, task.x, task.z); + break; + case ConvertTaskType::PLAYER: convertPlayer(task.file); break; + case ConvertTaskType::CONVERT_BLOCKS_DATA: + convertBlocksData(task.x, task.z, *report); + break; } } @@ -128,7 +267,7 @@ void WorldConverter::convertNext() { if (tasks.empty()) { throw std::runtime_error("no more regions to convert"); } - convert_task task = tasks.front(); + ConvertTask task = tasks.front(); tasks.pop(); tasksDone++; @@ -155,8 +294,22 @@ bool WorldConverter::isActive() const { } void WorldConverter::write() { - logger.info() << "writing world"; - wfile->write(nullptr, content); + logger.info() << "applying changes"; + + auto patch = dv::object(); + switch (mode) { + case ConvertMode::UPGRADE: + patch["region-version"] = REGION_FORMAT_VERSION; + break; + case ConvertMode::REINDEX: + WorldFiles::createContentIndicesCache(content->getIndices(), patch); + break; + case ConvertMode::BLOCK_FIELDS: + WorldFiles::createBlockFieldsIndices(content->getIndices(), patch); + break; + } + wfile->patchIndicesFile(patch); + wfile->write(nullptr, nullptr); } void WorldConverter::waitForEnd() { diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index c16151e8..b7852c97 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -6,40 +6,77 @@ #include "delegates.hpp" #include "interfaces/Task.hpp" +#include "files/world_regions_fwd.hpp" #include "typedefs.hpp" namespace fs = std::filesystem; class Content; -class ContentLUT; +class ContentReport; class WorldFiles; -enum class convert_task_type { region, player }; +enum class ConvertTaskType { + /// @brief rewrite voxels region indices + VOXELS, + /// @brief rewrite inventories region indices + INVENTORIES, + /// @brief rewrite player + PLAYER, + /// @brief refresh region file version + UPGRADE_REGION, + /// @brief convert blocks data to updated layouts + CONVERT_BLOCKS_DATA, +}; -struct convert_task { - convert_task_type type; +struct ConvertTask { + ConvertTaskType type; fs::path file; + + /// @brief region coords + int x, z; + RegionLayerIndex layer; +}; + +enum class ConvertMode { + UPGRADE, + REINDEX, + BLOCK_FIELDS, }; class WorldConverter : public Task { std::shared_ptr wfile; - std::shared_ptr const lut; + std::shared_ptr const report; const Content* const content; - std::queue tasks; + std::queue tasks; runnable onComplete; uint tasksDone = 0; + ConvertMode mode; + void upgradeRegion( + const fs::path& file, int x, int z, RegionLayerIndex layer) const; void convertPlayer(const fs::path& file) const; - void convertRegion(const fs::path& file) const; + void convertVoxels(const fs::path& file, int x, int z) const; + void convertInventories(const fs::path& file, int x, int z) const; + void convertBlocksData(int x, int z, const ContentReport& report) const; + + void addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType + ); + + void createUpgradeTasks(); + void createConvertTasks(); + void createBlockFieldsConvertTasks(); public: WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr lut + std::shared_ptr report, + ConvertMode mode ); ~WorldConverter(); - void convert(const convert_task& task) const; + void convert(const ConvertTask& task) const; void convertNext(); void setOnComplete(runnable callback); void write(); @@ -54,8 +91,9 @@ public: static std::shared_ptr startTask( const std::shared_ptr& worldFiles, const Content* content, - const std::shared_ptr& lut, + const std::shared_ptr& report, const runnable& onDone, + ConvertMode mode, bool multithreading ); }; diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 70eb1752..c99ee8ef 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -21,9 +21,11 @@ #include "objects/EntityDef.hpp" #include "objects/Player.hpp" #include "physics/Hitbox.hpp" +#include "data/StructLayout.hpp" #include "settings.hpp" #include "typedefs.hpp" #include "util/data_io.hpp" +#include "util/stringutil.hpp" #include "voxels/Block.hpp" #include "voxels/Chunk.hpp" #include "voxels/voxel.hpp" @@ -73,7 +75,9 @@ fs::path WorldFiles::getPacksFile() const { return directory / fs::path("packs.list"); } -void WorldFiles::write(const World* world, const Content* content) { +void WorldFiles::write( + const World* world, const Content* content +) { if (world) { writeWorldInfo(world->getInfo()); if (!fs::exists(getPacksFile())) { @@ -83,9 +87,10 @@ void WorldFiles::write(const World* world, const Content* content) { if (generatorTestMode) { return; } - - writeIndices(content->getIndices()); - regions.write(); + if (content) { + writeIndices(content->getIndices()); + } + regions.writeAll(); } void WorldFiles::writePacks(const std::vector& packs) { @@ -107,11 +112,33 @@ static void write_indices( } } -void WorldFiles::writeIndices(const ContentIndices* indices) { - dv::value root = dv::object(); +void WorldFiles::createContentIndicesCache( + const ContentIndices* indices, dv::value& root +) { write_indices(indices->blocks, root.list("blocks")); write_indices(indices->items, root.list("items")); write_indices(indices->entities, root.list("entities")); +} + +void WorldFiles::createBlockFieldsIndices( + const ContentIndices* indices, dv::value& root +) { + auto& structsMap = root.object("blocks-data"); + for (const auto* def : indices->blocks.getIterable()) { + if (def->dataStruct == nullptr) { + continue; + } + structsMap[def->name] = def->dataStruct->serialize(); + } +} + +void WorldFiles::writeIndices(const ContentIndices* indices) { + dv::value root = dv::object(); + root["region-version"] = REGION_FORMAT_VERSION; + + createContentIndicesCache(indices, root); + createBlockFieldsIndices(indices, root); + files::write_json(getIndicesFile(), root); } @@ -164,6 +191,20 @@ bool WorldFiles::readResourcesData(const Content* content) { return true; } +void WorldFiles::patchIndicesFile(const dv::value& map) { + fs::path file = getIndicesFile(); + if (!fs::is_regular_file(file)) { + logger.error() << file.filename().u8string() << " does not exists"; + return; + } + auto root = files::read_json(file); + for (const auto& [key, value] : map.asObject()) { + logger.info() << "patching indices.json: update " << util::quote(key); + root[key] = value; + } + files::write_json(file, root, true); +} + static void erase_pack_indices(dv::value& root, const std::string& id) { auto prefix = id + ":"; auto& blocks = root["blocks"]; diff --git a/src/files/WorldFiles.hpp b/src/files/WorldFiles.hpp index e0e78920..7f72e146 100644 --- a/src/files/WorldFiles.hpp +++ b/src/files/WorldFiles.hpp @@ -51,6 +51,15 @@ public: std::optional readWorldInfo(); bool readResourcesData(const Content* content); + static void createContentIndicesCache( + const ContentIndices* indices, dv::value& root + ); + static void createBlockFieldsIndices( + const ContentIndices* indices, dv::value& root + ); + + void patchIndicesFile(const dv::value& map); + /// @brief Write all unsaved data to world files /// @param world target world /// @param content world content @@ -63,8 +72,6 @@ public: /// @return world folder fs::path getFolder() const; - static const inline std::string WORLD_FILE = "world.json"; - WorldRegions& getRegions() { return regions; } @@ -72,4 +79,6 @@ public: bool doesWriteLights() const { return doWriteLights; } + + static const inline std::string WORLD_FILE = "world.json"; }; diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 93d1b232..a96fc61b 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -4,59 +4,23 @@ #include #include +#include "debug/Logger.hpp" #include "coders/byte_utils.hpp" #include "coders/rle.hpp" +#include "coders/binary_json.hpp" #include "items/Inventory.hpp" #include "maths/voxmaths.hpp" #include "util/data_io.hpp" -#include "coders/binary_json.hpp" #define REGION_FORMAT_MAGIC ".VOXREG" -regfile::regfile(fs::path filename) : file(std::move(filename)) { - if (file.length() < REGION_HEADER_SIZE) - throw std::runtime_error("incomplete region file header"); - char header[REGION_HEADER_SIZE]; - file.read(header, REGION_HEADER_SIZE); - - // avoid of use strcmp_s - if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) != - REGION_FORMAT_MAGIC) { - throw std::runtime_error("invalid region file magic number"); - } - version = header[8]; - if (static_cast(version) > REGION_FORMAT_VERSION) { - throw illegal_region_format( - "region format " + std::to_string(version) + " is not supported" - ); - } -} - -std::unique_ptr regfile::read(int index, uint32_t& length) { - size_t file_size = file.length(); - size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; - - uint32_t offset; - file.seekg(table_offset + index * 4); - file.read(reinterpret_cast(&offset), 4); - offset = dataio::read_int32_big(reinterpret_cast(&offset), 0); - if (offset == 0) { - return nullptr; - } - - file.seekg(offset); - file.read(reinterpret_cast(&offset), 4); - length = dataio::read_int32_big(reinterpret_cast(&offset), 0); - auto data = std::make_unique(length); - file.read(reinterpret_cast(data.get()), length); - return data; -} +static debug::Logger logger("world-regions"); WorldRegion::WorldRegion() : chunksData( std::make_unique[]>(REGION_CHUNKS_COUNT) ), - sizes(std::make_unique(REGION_CHUNKS_COUNT)) { + sizes(std::make_unique(REGION_CHUNKS_COUNT)) { } WorldRegion::~WorldRegion() = default; @@ -72,293 +36,89 @@ std::unique_ptr* WorldRegion::getChunks() const { return chunksData.get(); } -uint32_t* WorldRegion::getSizes() const { +glm::u32vec2* WorldRegion::getSizes() const { return sizes.get(); } -void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) { +void WorldRegion::put( + uint x, uint z, std::unique_ptr data, uint32_t size, uint32_t srcSize +) { size_t chunk_index = z * REGION_SIZE + x; - chunksData[chunk_index].reset(data); - sizes[chunk_index] = size; + chunksData[chunk_index] = std::move(data); + sizes[chunk_index] = glm::u32vec2(size, srcSize); } ubyte* WorldRegion::getChunkData(uint x, uint z) { return chunksData[z * REGION_SIZE + x].get(); } -uint WorldRegion::getChunkDataSize(uint x, uint z) { +glm::u32vec2 WorldRegion::getChunkDataSize(uint x, uint z) { return sizes[z * REGION_SIZE + x]; } WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { - for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) { - layers[i].layer = i; + for (size_t i = 0; i < REGION_LAYERS_COUNT; i++) { + layers[i].layer = static_cast(i); } - layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions"); - layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights"); + auto& voxels = layers[REGION_LAYER_VOXELS]; + voxels.folder = directory / fs::path("regions"); + voxels.compression = compression::Method::EXTRLE16; + + auto& lights = layers[REGION_LAYER_LIGHTS]; + lights.folder = directory / fs::path("lights"); + lights.compression = compression::Method::EXTRLE8; + layers[REGION_LAYER_INVENTORIES].folder = directory / fs::path("inventories"); layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities"); + + auto& blocksData = layers[REGION_LAYER_BLOCKS_DATA]; + blocksData.folder = directory / fs::path("blocksdata"); } WorldRegions::~WorldRegions() = default; -WorldRegion* WorldRegions::getRegion(int x, int z, int layer) { - RegionsLayer& regions = layers[layer]; - std::lock_guard lock(regions.mutex); - auto found = regions.regions.find(glm::ivec2(x, z)); - if (found == regions.regions.end()) { - return nullptr; - } - return found->second.get(); -} - -WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) { - if (auto region = getRegion(x, z, layer)) { - return region; - } - RegionsLayer& regions = layers[layer]; - std::lock_guard lock(regions.mutex); - auto region_ptr = std::make_unique(); - auto region = region_ptr.get(); - regions.regions[{x, z}] = std::move(region_ptr); - return region; -} - -std::unique_ptr WorldRegions::compress( - const ubyte* src, size_t srclen, size_t& len -) { - auto buffer = bufferPool.get(); - auto bytes = buffer.get(); - - len = extrle::encode(src, srclen, bytes); - auto data = std::make_unique(len); - for (size_t i = 0; i < len; i++) { - data[i] = bytes[i]; - } - return data; -} - -std::unique_ptr WorldRegions::decompress( - const ubyte* src, size_t srclen, size_t dstlen -) { - auto decompressed = std::make_unique(dstlen); - extrle::decode(src, srclen, decompressed.get()); - return decompressed; -} - -inline void calc_reg_coords( - int x, int z, int& regionX, int& regionZ, int& localX, int& localZ -) { - regionX = floordiv(x, REGION_SIZE); - regionZ = floordiv(z, REGION_SIZE); - localX = x - (regionX * REGION_SIZE); - localZ = z - (regionZ * REGION_SIZE); -} - -std::unique_ptr WorldRegions::readChunkData( - int x, int z, uint32_t& length, regfile* rfile -) { - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - int chunkIndex = localZ * REGION_SIZE + localX; - return rfile->read(chunkIndex, length); -} - -/// @brief Read missing chunks data (null pointers) from region file -void WorldRegions::fetchChunks( - WorldRegion* region, int x, int z, regfile* file -) { - auto* chunks = region->getChunks(); - uint32_t* sizes = region->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; - int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; - if (chunks[i] == nullptr) { - chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file); - } - } -} - -ubyte* WorldRegions::getData(int x, int z, int layer, uint32_t& size) { - if (generatorTestMode) { - return nullptr; - } - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); - ubyte* data = region->getChunkData(localX, localZ); - if (data == nullptr) { - auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer)); - if (regfile != nullptr) { - data = readChunkData(x, z, size, regfile.get()).release(); - } - if (data != nullptr) { - region->put(localX, localZ, data, size); - } - } - if (data != nullptr) { - size = region->getChunkDataSize(localX, localZ); - return data; - } - return nullptr; -} - -regfile_ptr WorldRegions::useRegFile(glm::ivec3 coord) { - auto* file = openRegFiles[coord].get(); - file->inUse = true; - return regfile_ptr(file, ®FilesCv); -} - -void WorldRegions::closeRegFile(glm::ivec3 coord) { - openRegFiles.erase(coord); - regFilesCv.notify_one(); -} - -// Marks regfile as used and unmarks when shared_ptr dies -regfile_ptr WorldRegions::getRegFile(glm::ivec3 coord, bool create) { - { - std::lock_guard lock(regFilesMutex); - const auto found = openRegFiles.find(coord); - if (found != openRegFiles.end()) { - if (found->second->inUse) { - throw std::runtime_error("regfile is currently in use"); - } - return useRegFile(found->first); - } - } - if (create) { - return createRegFile(coord); - } - return nullptr; -} - -regfile_ptr WorldRegions::createRegFile(glm::ivec3 coord) { - fs::path file = - layers[coord[2]].folder / getRegionFilename(coord[0], coord[1]); - if (!fs::exists(file)) { - return nullptr; - } - if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { - std::unique_lock lock(regFilesMutex); - while (true) { - bool closed = false; - // FIXME: bad choosing algorithm - for (auto& entry : openRegFiles) { - if (!entry.second->inUse) { - closeRegFile(entry.first); - closed = true; - break; - } - } - if (closed) { - break; - } - // notified when any regfile gets out of use or closed - regFilesCv.wait(lock); - } - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } else { - std::lock_guard lock(regFilesMutex); - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } -} - -fs::path WorldRegions::getRegionFilename(int x, int z) const { - return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); -} - -void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry) { - fs::path filename = layers[layer].folder / getRegionFilename(x, z); - - glm::ivec3 regcoord(x, z, layer); - if (auto regfile = getRegFile(regcoord, false)) { - fetchChunks(entry, x, z, regfile.get()); - - std::lock_guard lock(regFilesMutex); - regfile.reset(); - closeRegFile(regcoord); - } - - char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; - header[8] = REGION_FORMAT_VERSION; - header[9] = 0; // flags - std::ofstream file(filename, std::ios::out | std::ios::binary); - file.write(header, REGION_HEADER_SIZE); - - size_t offset = REGION_HEADER_SIZE; - char intbuf[4] {}; - uint offsets[REGION_CHUNKS_COUNT] {}; - - auto* region = entry->getChunks(); - uint32_t* sizes = entry->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - ubyte* chunk = region[i].get(); - if (chunk == nullptr) { - offsets[i] = 0; - } else { - offsets[i] = offset; - - size_t compressedSize = sizes[i]; - dataio::write_int32_big( - compressedSize, reinterpret_cast(intbuf), 0 - ); - offset += 4 + compressedSize; - - file.write(intbuf, 4); - file.write(reinterpret_cast(chunk), compressedSize); - } - } - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - dataio::write_int32_big( - offsets[i], reinterpret_cast(intbuf), 0 - ); - file.write(intbuf, 4); - } -} - -void WorldRegions::writeRegions(int layer) { - for (auto& it : layers[layer].regions) { +void RegionsLayer::writeAll() { + for (auto& it : regions) { WorldRegion* region = it.second.get(); if (region->getChunks() == nullptr || !region->isUnsaved()) { continue; } - glm::ivec2 key = it.first; - writeRegion(key[0], key[1], layer, region); + const auto& key = it.first; + writeRegion(key[0], key[1], region); } } void WorldRegions::put( int x, int z, - int layer, + RegionLayerIndex layerid, std::unique_ptr data, - size_t size, - bool rle + size_t srcSize ) { - if (rle) { - size_t compressedSize; - auto compressed = compress(data.get(), size, compressedSize); - put(x, z, layer, std::move(compressed), compressedSize, false); - return; - } + size_t size = srcSize; + auto& layer = layers[layerid]; int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); + WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); - region->put(localX, localZ, data.release(), size); + + if (data == nullptr) { + region->put(localX, localZ, nullptr, 0, 0); + return; + } + + if (layer.compression != compression::Method::NONE) { + data = compression::compress( + data.get(), size, size, layer.compression); + } + region->put(localX, localZ, std::move(data), size, srcSize); } static std::unique_ptr write_inventories( - Chunk* chunk, uint& datasize + const ChunkInventoriesMap& inventories, uint32_t& datasize ) { - auto& inventories = chunk->inventories; ByteBuilder builder; builder.putInt32(inventories.size()); for (auto& entry : inventories) { @@ -375,7 +135,22 @@ static std::unique_ptr write_inventories( return data; } -/// @brief Store chunk data (voxels and lights) in region (existing or new) +static ChunkInventoriesMap load_inventories(const ubyte* src, uint32_t size) { + ChunkInventoriesMap inventories; + ByteReader reader(src, size); + auto count = reader.getInt32(); + for (int i = 0; i < count; i++) { + uint index = reader.getInt32(); + uint size = reader.getInt32(); + auto map = json::from_binary(reader.pointer(), size); + reader.skip(size); + auto inv = std::make_shared(0, 0); + inv->deserialize(map); + inventories[index] = inv; + } + return inventories; +} + void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { assert(chunk != nullptr); if (!chunk->flags.lighted) { @@ -393,8 +168,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_VOXELS, chunk->encode(), - CHUNK_DATA_LEN, - true); + CHUNK_DATA_LEN); // Writing lights cache if (doWriteLights && chunk->flags.lighted) { @@ -402,19 +176,17 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_LIGHTS, chunk->lightmap.encode(), - LIGHTMAP_DATA_LEN, - true); + LIGHTMAP_DATA_LEN); } // Writing block inventories if (!chunk->inventories.empty()) { uint datasize; - auto data = write_inventories(chunk, datasize); + auto data = write_inventories(chunk->inventories, datasize); put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES, std::move(data), - datasize, - false); + datasize); } // Writing entities if (!entitiesData.empty()) { @@ -426,71 +198,164 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_ENTITIES, std::move(data), - entitiesData.size(), - false); + entitiesData.size()); + } + // Writing blocks data + if (chunk->flags.blocksData) { + auto bytes = chunk->blocksMetadata.serialize(); + put(chunk->x, + chunk->z, + REGION_LAYER_BLOCKS_DATA, + bytes.release(), + bytes.size()); } } -std::unique_ptr WorldRegions::getChunk(int x, int z) { +std::unique_ptr WorldRegions::getVoxels(int x, int z) { uint32_t size; - auto* data = getData(x, z, REGION_LAYER_VOXELS, size); + uint32_t srcSize; + auto& layer = layers[REGION_LAYER_VOXELS]; + auto* data = layer.getData(x, z, size, srcSize); if (data == nullptr) { return nullptr; } - return decompress(data, size, CHUNK_DATA_LEN); + assert(srcSize == CHUNK_DATA_LEN); + return compression::decompress(data, size, srcSize, layer.compression); } -/// @brief Get cached lights for chunk at x,z -/// @return lights data or nullptr std::unique_ptr WorldRegions::getLights(int x, int z) { uint32_t size; - auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size); + uint32_t srcSize; + auto& layer = layers[REGION_LAYER_LIGHTS]; + auto* bytes = layer.getData(x, z, size, srcSize); if (bytes == nullptr) { return nullptr; } - auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN); + auto data = compression::decompress( + bytes, size, srcSize, layer.compression + ); + assert(srcSize == LIGHTMAP_DATA_LEN); return Lightmap::decode(data.get()); } -chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { - chunk_inventories_map meta; +ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) { uint32_t bytesSize; - const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize); - if (data == nullptr) { - return meta; + uint32_t srcSize; + auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize, srcSize); + if (bytes == nullptr) { + return {}; } - ByteReader reader(data, bytesSize); - auto count = reader.getInt32(); - for (int i = 0; i < count; i++) { - uint index = reader.getInt32(); - uint size = reader.getInt32(); - auto map = json::from_binary(reader.pointer(), size); - reader.skip(size); - auto inv = std::make_shared(0, 0); - inv->deserialize(map); - meta[index] = inv; + return load_inventories(bytes, bytesSize); +} + +BlocksMetadata WorldRegions::getBlocksData(int x, int z) { + uint32_t bytesSize; + uint32_t srcSize; + auto bytes = layers[REGION_LAYER_BLOCKS_DATA].getData(x, z, bytesSize, srcSize); + if (bytes == nullptr) { + return {}; + } + BlocksMetadata heap; + heap.deserialize(bytes, bytesSize); + return heap; +} + +void WorldRegions::processInventories(int x, int z, const InventoryProc& func) { + processRegion(x, z, REGION_LAYER_INVENTORIES, + [=](std::unique_ptr data, uint32_t* size) { + auto inventories = load_inventories(data.get(), *size); + for (const auto& [_, inventory] : inventories) { + func(inventory.get()); + } + return write_inventories(inventories, *size); + }); +} + +void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) { + auto& voxLayer = layers[REGION_LAYER_VOXELS]; + auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA]; + if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) { + throw std::runtime_error("not implemented for in-memory regions"); + } + auto datRegfile = datLayer.getRegFile({x, z}); + if (datRegfile == nullptr) { + throw std::runtime_error("could not open region file"); + } + auto voxRegfile = voxLayer.getRegFile({x, z}); + if (voxRegfile == nullptr) { + logger.warning() << "missing voxels region - discard blocks data for " + << x << "_" << z; + deleteRegion(REGION_LAYER_BLOCKS_DATA, x, z); + return; + } + for (uint cz = 0; cz < REGION_SIZE; cz++) { + for (uint cx = 0; cx < REGION_SIZE; cx++) { + int gx = cx + x * REGION_SIZE; + int gz = cz + z * REGION_SIZE; + + uint32_t datLength; + uint32_t datSrcSize; + auto datData = RegionsLayer::readChunkData( + gx, gz, datLength, datSrcSize, datRegfile.get() + ); + if (datData == nullptr) { + continue; + } + uint32_t voxLength; + uint32_t voxSrcSize; + auto voxData = RegionsLayer::readChunkData( + gx, gz, voxLength, voxSrcSize, voxRegfile.get() + ); + if (voxData == nullptr) { + logger.warning() + << "missing voxels for chunk (" << gx << ", " << gz << ")"; + put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0); + continue; + } + voxData = compression::decompress( + voxData.get(), voxLength, voxSrcSize, voxLayer.compression + ); + + BlocksMetadata blocksData; + blocksData.deserialize(datData.get(), datLength); + try { + func(&blocksData, std::move(voxData)); + } catch (const std::exception& err) { + logger.error() << "an error ocurred while processing blocks " + "data in chunk (" << gx << ", " << gz << "): " << err.what(); + blocksData = {}; + } + auto bytes = blocksData.serialize(); + put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size()); + } } - return meta; } dv::value WorldRegions::fetchEntities(int x, int z) { + if (generatorTestMode) { + return nullptr; + } uint32_t bytesSize; - const ubyte* data = getData(x, z, REGION_LAYER_ENTITIES, bytesSize); + uint32_t srcSize; + const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize, srcSize); if (data == nullptr) { return nullptr; } auto map = json::from_binary(data, bytesSize); - if (map.size() == 0) { + if (map.empty()) { return nullptr; } return map; } -void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - if (getRegion(x, z, REGION_LAYER_VOXELS)) { +void WorldRegions::processRegion( + int x, int z, RegionLayerIndex layerid, const RegionProc& func +) { + auto& layer = layers[layerid]; + if (layer.getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } - auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS)); + auto regfile = layer.getRegFile({x, z}); if (regfile == nullptr) { throw std::runtime_error("could not open region file"); } @@ -499,31 +364,50 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { int gx = cx + x * REGION_SIZE; int gz = cz + z * REGION_SIZE; uint32_t length; - auto data = readChunkData(gx, gz, length, regfile.get()); + uint32_t srcSize; + auto data = + RegionsLayer::readChunkData(gx, gz, length, srcSize, regfile.get()); if (data == nullptr) { continue; } - data = decompress(data.get(), length, CHUNK_DATA_LEN); - if (func(data.get())) { - put(gx, - gz, - REGION_LAYER_VOXELS, - std::move(data), - CHUNK_DATA_LEN, - true); + if (layer.compression != compression::Method::NONE) { + data = compression::decompress( + data.get(), length, srcSize, layer.compression + ); + } else { + srcSize = length; + } + if (auto writeData = func(std::move(data), &srcSize)) { + put(gx, gz, layerid, std::move(writeData), srcSize); } } } } -fs::path WorldRegions::getRegionsFolder(int layer) const { - return layers[layer].folder; +const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const { + return layers[layerid].folder; } -void WorldRegions::write() { +fs::path WorldRegions::getRegionFilePath(RegionLayerIndex layerid, int x, int z) const { + return layers[layerid].getRegionFilePath(x, z); +} + +void WorldRegions::writeAll() { for (auto& layer : layers) { fs::create_directories(layer.folder); - writeRegions(layer.layer); + layer.writeAll(); + } +} + +void WorldRegions::deleteRegion(RegionLayerIndex layerid, int x, int z) { + auto& layer = layers[layerid]; + if (layer.getRegFile({x, z}, false)) { + throw std::runtime_error("region file is currently in use"); + } + auto file = layer.getRegionFilePath(x, z); + if (fs::exists(file)) { + logger.info() << "remove region file " << file.u8string(); + fs::remove(file); } } diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index 9923ee05..ad0bc584 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -11,7 +11,10 @@ #include "typedefs.hpp" #include "util/BufferPool.hpp" #include "voxels/Chunk.hpp" +#include "maths/voxmaths.hpp" +#include "coders/compression.hpp" #include "files.hpp" +#include "world_regions_fwd.hpp" #define GLM_ENABLE_EXPERIMENTAL #include @@ -20,16 +23,9 @@ namespace fs = std::filesystem; inline constexpr uint REGION_HEADER_SIZE = 10; -inline constexpr uint REGION_LAYER_VOXELS = 0; -inline constexpr uint REGION_LAYER_LIGHTS = 1; -inline constexpr uint REGION_LAYER_INVENTORIES = 2; -inline constexpr uint REGION_LAYER_ENTITIES = 3; - inline constexpr uint REGION_SIZE_BIT = 5; inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); -inline constexpr uint REGION_FORMAT_VERSION = 2; -inline constexpr uint MAX_OPEN_REGION_FILES = 16; class illegal_region_format : public std::runtime_error { public: @@ -40,21 +36,21 @@ public: class WorldRegion { std::unique_ptr[]> chunksData; - std::unique_ptr sizes; + std::unique_ptr sizes; bool unsaved = false; public: WorldRegion(); ~WorldRegion(); - void put(uint x, uint z, ubyte* data, uint32_t size); + void put(uint x, uint z, std::unique_ptr data, uint32_t size, uint32_t srcSize); ubyte* getChunkData(uint x, uint z); - uint getChunkDataSize(uint x, uint z); + glm::u32vec2 getChunkDataSize(uint x, uint z); void setUnsaved(bool unsaved); bool isUnsaved() const; std::unique_ptr* getChunks() const; - uint32_t* getSizes() const; + glm::u32vec2* getSizes() const; }; struct regfile { @@ -65,19 +61,15 @@ struct regfile { regfile(fs::path filename); regfile(const regfile&) = delete; - std::unique_ptr read(int index, uint32_t& length); + std::unique_ptr read(int index, uint32_t& size, uint32_t& srcSize); }; -using regionsmap = std::unordered_map>; -using regionproc = std::function; - -struct RegionsLayer { - int layer; - fs::path folder; - regionsmap regions; - std::mutex mutex; -}; +using RegionsMap = std::unordered_map>; +using RegionProc = std::function(std::unique_ptr,uint32_t*)>; +using InventoryProc = std::function; +using BlockDataProc = std::function)>; +/// @brief Region file pointer keeping inUse flag on until destroyed class regfile_ptr { regfile* file; std::condition_variable* cv; @@ -115,58 +107,80 @@ public: } }; -class WorldRegions { - fs::path directory; - std::unordered_map> openRegFiles; +inline void calc_reg_coords( + int x, int z, int& regionX, int& regionZ, int& localX, int& localZ +) { + regionX = floordiv(x, REGION_SIZE); + regionZ = floordiv(z, REGION_SIZE); + localX = x - (regionX * REGION_SIZE); + localZ = z - (regionZ * REGION_SIZE); +} + +struct RegionsLayer { + /// @brief Layer index + RegionLayerIndex layer; + + /// @brief Regions layer folder + fs::path folder; + + compression::Method compression = compression::Method::NONE; + + /// @brief In-memory regions data + RegionsMap regions; + + /// @brief In-memory regions map mutex + std::mutex mapMutex; + + /// @brief Open region files map + std::unordered_map> openRegFiles; + + /// @brief Open region files map mutex std::mutex regFilesMutex; std::condition_variable regFilesCv; - RegionsLayer layers[4] {}; - util::BufferPool bufferPool { - std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2}; - WorldRegion* getRegion(int x, int z, int layer); - WorldRegion* getOrCreateRegion(int x, int z, int layer); + [[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); + [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord); + regfile_ptr createRegFile(glm::ivec2 coord); + void closeRegFile(glm::ivec2 coord); - /// @brief Compress buffer with extrle - /// @param src source buffer - /// @param srclen length of the source buffer - /// @param len (out argument) length of result buffer - /// @return compressed bytes array - std::unique_ptr compress( - const ubyte* src, size_t srclen, size_t& len - ); + WorldRegion* getRegion(int x, int z); + WorldRegion* getOrCreateRegion(int x, int z); - /// @brief Decompress buffer with extrle - /// @param src compressed buffer - /// @param srclen length of compressed buffer - /// @param dstlen max expected length of source buffer - /// @return decompressed bytes array - std::unique_ptr decompress( - const ubyte* src, size_t srclen, size_t dstlen - ); + fs::path getRegionFilePath(int x, int z) const; - std::unique_ptr readChunkData( - int x, int y, uint32_t& length, regfile* file - ); - - void fetchChunks(WorldRegion* region, int x, int y, regfile* file); - - ubyte* getData(int x, int z, int layer, uint32_t& size); - - regfile_ptr getRegFile(glm::ivec3 coord, bool create = true); - void closeRegFile(glm::ivec3 coord); - regfile_ptr useRegFile(glm::ivec3 coord); - regfile_ptr createRegFile(glm::ivec3 coord); - - fs::path getRegionFilename(int x, int y) const; - - void writeRegions(int layer); + /// @brief Get chunk data. Read from file if not loaded yet. + /// @param x chunk x coord + /// @param z chunk z coord + /// @param size [out] compressed chunk data length + /// @param size [out] source chunk data length + /// @return nullptr if no saved chunk data found + [[nodiscard]] ubyte* getData(int x, int z, uint32_t& size, uint32_t& srcSize); /// @brief Write or rewrite region file /// @param x region X /// @param z region Z - /// @param layer regions layer - void writeRegion(int x, int y, int layer, WorldRegion* entry); + void writeRegion(int x, int y, WorldRegion* entry); + + /// @brief Write all unsaved regions to files + void writeAll(); + + /// @brief Read chunk data from region file + /// @param x chunk x coord + /// @param z chunk z coord + /// @param size [out] compressed chunk data length + /// @param srcSize [out] source chunk data length + /// @param rfile region file + /// @return nullptr if chunk is not present in region file + [[nodiscard]] static std::unique_ptr readChunkData( + int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile + ); +}; + +class WorldRegions { + /// @brief World directory + fs::path directory; + + RegionsLayer layers[REGION_LAYERS_COUNT] {}; public: bool generatorTestMode = false; bool doWriteLights = true; @@ -184,26 +198,57 @@ public: /// @param layer regions layer /// @param data target data /// @param size data size - /// @param rle compress with ext-RLE void put( int x, int z, - int layer, + RegionLayerIndex layer, std::unique_ptr data, - size_t size, - bool rle + size_t size ); - std::unique_ptr getChunk(int x, int z); + /// @brief Get chunk voxels data + /// @param x chunk.x + /// @param z chunk.z + /// @return voxels data buffer or nullptr + std::unique_ptr getVoxels(int x, int z); + + /// @brief Get cached lights for chunk at x,z + /// @return lights data or nullptr std::unique_ptr getLights(int x, int z); - chunk_inventories_map fetchInventories(int x, int z); + + ChunkInventoriesMap fetchInventories(int x, int z); + + BlocksMetadata getBlocksData(int x, int z); + + /// @brief Load saved entities data for chunk + /// @param x chunk.x + /// @param z chunk.z + /// @return map with entities list as "data" dv::value fetchEntities(int x, int z); - void processRegionVoxels(int x, int z, const regionproc& func); + /// @brief Load, process and save processed region chunks data + /// @param x region X + /// @param z region Z + /// @param layerid regions layer index + /// @param func processing callback + void processRegion( + int x, int z, RegionLayerIndex layerid, const RegionProc& func); - fs::path getRegionsFolder(int layer) const; + void processInventories(int x, int z, const InventoryProc& func); - void write(); + void processBlocksData(int x, int z, const BlockDataProc& func); + + /// @brief Get regions directory by layer index + /// @param layerid layer index + /// @return directory path + const fs::path& getRegionsFolder(RegionLayerIndex layerid) const; + + fs::path getRegionFilePath(RegionLayerIndex layerid, int x, int z) const; + + /// @brief Write all region layers + void writeAll(); + + void deleteRegion(RegionLayerIndex layerid, int x, int z); /// @brief Extract X and Z from 'X_Z.bin' region file name. /// @param name source region file name diff --git a/src/files/compatibility.cpp b/src/files/compatibility.cpp new file mode 100644 index 00000000..0ffc97fe --- /dev/null +++ b/src/files/compatibility.cpp @@ -0,0 +1,110 @@ +#include "compatibility.hpp" + +#include + +#include "constants.hpp" +#include "voxels/voxel.hpp" +#include "coders/compression.hpp" +#include "coders/byte_utils.hpp" +#include "lighting/Lightmap.hpp" +#include "util/data_io.hpp" + +static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4; +static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4; + +static util::Buffer convert_voxels_1to2(const ubyte* buffer, uint32_t size) { + auto data = compression::decompress( + buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8); + + util::Buffer dstBuffer(VOXELS_DATA_SIZE_V2); + auto dst = reinterpret_cast(dstBuffer.data()); + + for (size_t i = 0; i < CHUNK_VOL; i++) { + ubyte bid1 = data[i]; + ubyte bid2 = data[CHUNK_VOL + i]; + + ubyte bst1 = data[CHUNK_VOL * 2 + i]; + ubyte bst2 = data[CHUNK_VOL * 3 + i]; + + dst[i] = + (static_cast(bid1) << 8) | static_cast(bid2); + dst[CHUNK_VOL + i] = ( + (static_cast(bst1) << 8) | + static_cast(bst2) + ); + } + size_t outLen; + auto compressed = compression::compress( + dstBuffer.data(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16); + return util::Buffer(std::move(compressed), outLen); +} + +util::Buffer compatibility::convert_region_2to3( + const util::Buffer& src, RegionLayerIndex layer +) { + const size_t REGION_CHUNKS = 1024; + const size_t HEADER_SIZE = 10; + const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t); + const ubyte COMPRESS_NONE = 0; + const ubyte COMPRESS_EXTRLE8 = 1; + const ubyte COMPRESS_EXTRLE16 = 2; + + const ubyte* const ptr = src.data(); + + ByteBuilder builder; + builder.putCStr(".VOXREG"); + builder.put(3); + switch (layer) { + case REGION_LAYER_VOXELS: builder.put(COMPRESS_EXTRLE16); break; + case REGION_LAYER_LIGHTS: builder.put(COMPRESS_EXTRLE8); break; + default: builder.put(COMPRESS_NONE); break; + } + + uint32_t offsets[REGION_CHUNKS] {}; + size_t chunkIndex = 0; + + auto tablePtr = reinterpret_cast( + ptr + src.size() - OFFSET_TABLE_SIZE + ); + + for (size_t i = 0; i < REGION_CHUNKS; i++) { + uint32_t srcOffset = dataio::be2h(tablePtr[i]); + if (srcOffset == 0) { + continue; + } + uint32_t size = *reinterpret_cast(ptr + srcOffset); + size = dataio::be2h(size); + + const ubyte* data = ptr + srcOffset + sizeof(uint32_t); + offsets[i] = builder.size(); + + switch (layer) { + case REGION_LAYER_VOXELS: { + auto dstdata = convert_voxels_1to2(data, size); + builder.putInt32(dstdata.size()); + builder.putInt32(VOXELS_DATA_SIZE_V2); + builder.put(dstdata.data(), dstdata.size()); + break; + } + case REGION_LAYER_LIGHTS: + builder.putInt32(size); + builder.putInt32(LIGHTMAP_DATA_LEN); + builder.put(data, size); + break; + case REGION_LAYER_ENTITIES: + case REGION_LAYER_INVENTORIES: + case REGION_LAYER_BLOCKS_DATA: { + builder.putInt32(size); + builder.putInt32(size); + builder.put(data, size); + break; + case REGION_LAYERS_COUNT: + throw std::invalid_argument("invalid enum"); + } + } + } + for (size_t i = 0; i < REGION_CHUNKS; i++) { + builder.putInt32(offsets[i]); + } + return util::Buffer(builder.build().data(), builder.size()); +} diff --git a/src/files/compatibility.hpp b/src/files/compatibility.hpp new file mode 100644 index 00000000..cf78b5bc --- /dev/null +++ b/src/files/compatibility.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "typedefs.hpp" +#include "util/Buffer.hpp" +#include "files/world_regions_fwd.hpp" + +namespace compatibility { + /// @brief Convert region file from version 2 to 3 + /// @see /doc/specs/region_file_spec.md + /// @param src region file source content + /// @return new region file content + util::Buffer convert_region_2to3( + const util::Buffer& src, RegionLayerIndex layer); +} diff --git a/src/files/files.cpp b/src/files/files.cpp index c1f74c63..5a84b27f 100644 --- a/src/files/files.cpp +++ b/src/files/files.cpp @@ -65,6 +65,12 @@ bool files::read(const fs::path& filename, char* data, size_t size) { return true; } +util::Buffer files::read_bytes_buffer(const fs::path& path) { + size_t size; + auto bytes = files::read_bytes(path, size); + return util::Buffer(std::move(bytes), size); +} + std::unique_ptr files::read_bytes( const fs::path& filename, size_t& length ) { diff --git a/src/files/files.hpp b/src/files/files.hpp index d6827dae..5055e5a9 100644 --- a/src/files/files.hpp +++ b/src/files/files.hpp @@ -8,6 +8,7 @@ #include "typedefs.hpp" #include "data/dv.hpp" +#include "util/Buffer.hpp" namespace fs = std::filesystem; @@ -56,6 +57,7 @@ namespace files { ); bool read(const fs::path&, char* data, size_t size); + util::Buffer read_bytes_buffer(const fs::path&); std::unique_ptr read_bytes(const fs::path&, size_t& length); std::vector read_bytes(const fs::path&); std::string read_string(const fs::path& filename); diff --git a/src/files/world_regions_fwd.hpp b/src/files/world_regions_fwd.hpp new file mode 100644 index 00000000..9938239d --- /dev/null +++ b/src/files/world_regions_fwd.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "typedefs.hpp" + +enum RegionLayerIndex : uint { + REGION_LAYER_VOXELS = 0, + REGION_LAYER_LIGHTS, + REGION_LAYER_INVENTORIES, + REGION_LAYER_ENTITIES, + REGION_LAYER_BLOCKS_DATA, + + REGION_LAYERS_COUNT +}; diff --git a/src/graphics/ui/gui_util.cpp b/src/graphics/ui/gui_util.cpp index 2045713a..70bd94f5 100644 --- a/src/graphics/ui/gui_util.cpp +++ b/src/graphics/ui/gui_util.cpp @@ -3,6 +3,7 @@ #include "elements/Label.hpp" #include "elements/Menu.hpp" #include "elements/Button.hpp" +#include "elements/TextBox.hpp" #include "gui_xml.hpp" #include "logic/scripting/scripting.hpp" @@ -77,3 +78,47 @@ void guiutil::confirm( menu->addPage("", panel); menu->setPage(""); } + +void guiutil::confirmWithMemo( + gui::GUI* gui, + const std::wstring& text, + const std::wstring& memo, + const runnable& on_confirm, + std::wstring yestext, + std::wstring notext) { + + if (yestext.empty()) yestext = langs::get(L"Yes"); + if (notext.empty()) notext = langs::get(L"No"); + + auto menu = gui->getMenu(); + auto panel = std::make_shared(glm::vec2(600, 500), glm::vec4(8.0f), 8.0f); + panel->setColor(glm::vec4(0.0f, 0.0f, 0.0f, 0.5f)); + panel->add(std::make_shared