Merge branch 'main' into heightmaps

doc/en/block-properties.md

@@ -142,3 +142,42 @@ Number of block inventory slots. Default - 0 (no inventory).
 ### *size*
 
 Array of three integers. Default value is `[1, 1, 1]`.
+
+## Block fields
+
+Block fields allow you to write more data unique to a specified voxel than the user bits allow.
+
+Block fields are declared in the following format:
+
+```json
+"fields": {
+    "name": {"type": "data_type"},
+    "array_name": {"type": "data_type", "length": "array_length"}
+}
+```
+
+In addition to `type` and `length`, the `convert-strategy` parameter determines the value conversion strategy when narrowing the data type.
+
+The parameter takes one of two values:
+- `reset` - a value that does not exists in the new range will be reset to 0
+- `clamp` - the value will be reduced to the closest one in the new range
+
+Example: the number 231 when changing the field type from int16 to int8:
+- in `reset` mode will turn into 0
+- in `clamp` mode will turn into 127
+
+Available data types:
+
+| Type    | Size      | Description            |
+| ------- | --------- | ---------------------- |
+| int8    | 1 byte    | signed integer 8 bits  |
+| int16   | 2 bytes   | signed integer 16 bits |
+| int32   | 4 bytes   | signed integer 32 bits |
+| int64   | 8 bytes   | integer signed 64 bits |
+| float32 | 4 bytes   | floating-point 32 bits |
+| float64 | 8 bytes   | floating-point 64 bits |
+| char    | 1 byte    | character              |
+
+- Currently, the total sum of the field sizes cannot exceed 240 bytes.
+- A field without an array length specification is equivalent to an array of 1 element.
+- A character array can be used to store UTF-8 strings.

doc/en/scripting/builtins/libblock.md

@@ -132,3 +132,29 @@ To use filter `dest` argument must be filled with some value(can be nil), it's d
 The function returns a table with the results or nil if the ray does not hit any block.
 
 The result will use the destination table instead of creating a new one if the optional argument specified.
+
+## Data fields
+
+```lua
+-- writes a value to the specified block field
+-- * throws an exception if the types are incompatible
+-- * throws an exception when array is out of bounds
+-- * does nothing if the block does not have the field
+block.set_field(
+    x: int, y: int, z: int,
+    name: str,
+    value: bool|int|number|string,
+    [optional] index: int = 0
+)
+
+-- returns the value written to the block field
+-- * returns nil if:
+--     1. the field does not exist
+--     2. no writes were made to any block field
+-- * throws an exception when array is out of bounds
+block.get_field(
+    x: int, y: int, z: int,
+    name: str,
+    [optional] index: int = 0
+) -> the stored value or nil
+```

doc/ru/block-properties.md

@@ -145,3 +145,44 @@
 ### Размер блока - *size*
 
 Массив из трех целых чисел. Значение по-умолчанию - `[1, 1, 1]`.
+
+## Поля блока
+
+Поля блоков позволяет записывать больше уникальных для конкретного блока данных, чем это позволяют пользовательские биты.
+
+Поля блока объявляются в следующем формате:
+
+```json
+"fields": {
+    "имя": {"type": "тип_данных"},
+    "имя_массива": {"type": "тип_данных", "length": "длина_массива"}
+}
+```
+
+Кроме `type` и `length` доступен параметр `convert-strategy` определяющий
+стратегию конвертации значения при сужении типа данных.
+
+Параметр принимает одно из двух значений:
+- `reset` - значение, не попадающее в новый диапазон, будет сброшено до 0
+- `clamp` - значение будет сведено к ближайшему в новом диапазоне
+
+Пример: число 231 при изменении типа поля с int16 до int8:
+- в режиме `reset` превратится в 0
+- в режиме `clamp` превратится в 127
+
+
+Доступные типы данных:
+
+| Тип     | Размер   | Описание                      |
+| ------- | -------- | ----------------------------- |
+| int8    | 1 байт   | целочисленный знаковый 8 бит  |
+| int16   | 2 байта  | целочисленный знаковый 16 бит |
+| int32   | 4 байта  | целочисленный знаковый 32 бит |
+| int64   | 8 байт   | целочисленный знаковый 64 бит |
+| float32 | 4 байта  | вещественный 32 бит           |
+| float64 | 8 байт   | вещественный 64 бит           |
+| char    | 1 байт   | символьный                    |
+
+- На данный момент общая сумма размеров полей не может превышать 240 байт.
+- Поле без указания длины массива эквивалентно массиву из 1 элемента.
+- Массив символьного типа может использоваться для хранения UTF-8 строк.

doc/ru/scripting/builtins/libblock.md

@@ -159,3 +159,29 @@ block.get_model(id: int) -> str
 -- возвращает массив из 6 текстур, назначенных на стороны блока
 block.get_textures(id: int) -> таблица строк
 ```
+
+## Поля данных
+
+```lua
+-- записывает значение в указанное поле блока
+-- * бросает исключение при несовместимости типов
+-- * бросает исключение при выходе за границы массива
+-- * ничего не делает при отсутствии поля у блока
+block.set_field(
+    x: int, y: int, z: int, 
+    name: str,
+    value: bool|int|number|string, 
+    [опционально] index: int = 0
+)
+
+-- возвращает значение записанное в поле блока
+-- * возвращает nil если:
+--     1. поле не существует
+--     2. ни в одно поле блока не было произведено записи
+-- * бросает исключение при выходе за границы массива
+block.get_field(
+    x: int, y: int, z: int, 
+    name: str, 
+    [опционально] index: int = 0
+) -> хранимое значение или nil
+```

doc/specs/binary_json_spec.md

@@ -49,9 +49,3 @@ int32     = 4byte
 int16     = 2byte
 byte      = %x00-FF
 ```
-
-## VoxelEngine format support
-
-Current implementation does not support types: bytes array, null, compressed document. 
-
-All unsupported types will be implemented in future.

doc/specs/outdated/region_file_spec_v2.md

@@ -0,0 +1,41 @@
+# Region File (version 2)
+
+File format BNF (RFC 5234):
+
+```bnf
+file    = header (*chunk) offsets   complete file
+header  = magic %x02 %x00           magic number, version and reserved
+                                    zero byte
+
+magic   = %x2E %x56 %x4F %x58       '.VOXREG\0'
+          %x52 %x45 %x47 %x00
+
+chunk   = int32 (*byte)             byte array with size prefix
+offsets = (1024*int32)              offsets table
+int32   = 4byte                     signed big-endian 32 bit integer
+byte    = %x00-FF                   8 bit unsigned integer
+```
+
+C struct visualization:
+
+```c
+typedef unsigned char byte;
+
+struct file {
+	// 10 bytes
+	struct {
+		char magic[8] = ".VOXREG";
+		byte version = 2;
+		byte reserved = 0;
+	} header;
+	
+	struct {
+		int32_t size; // byteorder: big-endian
+		byte* data;
+	} chunks[1024]; // file does not contain zero sizes for missing chunks
+	
+	int32_t offsets[1024]; // byteorder: big-endian
+};
+```
+
+Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size).

doc/specs/outdated/region_voxels_chunk_spec_v1.md

@@ -0,0 +1,26 @@
+# Voxels Chunk (version 1)
+
+Voxel regions layer chunk structure.
+
+Values are separated for extRLE8 compression efficiency.
+
+File format BNF (RFC 5234):
+
+```bnf
+chunk    = (65536*byte)    block indices (most significant bytes)
+           (65536*byte)    block indices (least significant bytes)
+           (65536*byte)    block states (most significant bytes)
+           (65536*byte)    block states (least significant bytes)
+
+byte     = %x00-FF         8 bit unsigned integer
+```
+
+65536 is number of voxels per chunk (16\*256\*16)
+
+## Block state
+
+Block state is encoded in 16 bits:
+- 0-2 bits (3) - block rotation index
+- 3-5 bits (3) - segment block bits
+- 6-7 bits (2) - reserved
+- 8-15 bits (8) - user bits

doc/specs/region_file_spec.md

@@ -0,0 +1,50 @@
+# Region File (version 3)
+
+File format BNF (RFC 5234):
+
+```bnf
+file    = header (*chunk) offsets   complete file
+header  = magic %x02 byte           magic number, version and compression
+                                    method
+
+magic   = %x2E %x56 %x4F %x58       '.VOXREG\0'
+          %x52 %x45 %x47 %x00
+
+chunk   = uint32 uint32 (*byte)     byte array with size and source size 
+                                    prefix where source size is 
+                                    decompressed chunk data size
+
+offsets = (1024*uint32)             offsets table
+int32   = 4byte                     unsigned big-endian 32 bit integer
+byte    = %x00-FF                   8 bit unsigned integer
+```
+
+C struct visualization:
+
+```c
+typedef unsigned char byte;
+
+struct file {
+	// 10 bytes
+	struct {
+		char magic[8] = ".VOXREG";
+		byte version = 3;
+		byte compression;
+	} header;
+	
+	struct {
+		uint32_t size; // byteorder: little-endian
+		uint32_t sourceSize; // byteorder: little-endian
+		byte* data;
+	} chunks[1024]; // file does not contain zero sizes for missing chunks
+	
+	uint32_t offsets[1024]; // byteorder: little-endian
+};
+```
+
+Offsets table contains chunks positions in file. 0 means that chunk is not present in the file. Minimal valid offset is 10 (header size).
+
+Available compression methods:
+0. no compression
+1. extRLE8
+2. extRLE16

doc/specs/region_voxels_chunk_spec.md

@@ -0,0 +1,23 @@
+# Voxels Chunk (version 2)
+
+IDs and states are separated for extRLE16 compression efficiency.
+
+File format BNF (RFC 5234):
+
+```bnf
+chunk    = (65536*uint16)  block ids
+           (65536*uint16)  block states
+
+uint16   = 2byte           16 bit little-endian unsigned integer
+byte     = %x00-FF         8 bit unsigned integer
+```
+
+65536 is number of voxels per chunk (16\*256\*16)
+
+## Block state
+
+Block state is encoded in 16 bits:
+- 0-2 bits (3) - block rotation index
+- 3-5 bits (3) - segment block bits
+- 6-7 bits (2) - reserved
+- 8-15 bits (8) - user bits

res/texts/en_US.txt

@@ -1,6 +1,9 @@
 # Menu
 menu.missing-content=Missing Content!
 world.convert-request=Content indices have changed! Convert world files?
+world.upgrade-request=World format is outdated! Convert world files?
+world.convert-with-loss=Convert world with data loss?
+world.convert-block-layouts=Blocks fields have changes! Convert world files?
 pack.remove-confirm=Do you want to erase all pack(s) content from the world forever?
 error.pack-not-found=Could not to find pack
 error.dependency-not-found=Dependency pack is not found

res/texts/ru_RU.txt

@@ -46,6 +46,9 @@ world.generators.default=Обычный
 world.generators.flat=Плоский
 world.Create World=Создать Мир
 world.convert-request=Есть изменения в индексах! Конвертировать мир?
+world.upgrade-request=Формат мира устарел! Конвертировать мир?
+world.convert-with-loss=Конвертировать мир с потерями?
+world.convert-block-layouts=Есть изменения в полях блоков! Конвертировать мир?
 world.delete-confirm=Удалить мир безвозвратно?
 
 # Настройки

src/coders/byte_utils.cpp

@@ -4,6 +4,8 @@
 #include <limits>
 #include <stdexcept>
 
+#include "util/data_io.hpp"
+
 void ByteBuilder::put(ubyte b) {
     buffer.push_back(b);
 }
@@ -30,28 +32,24 @@ void ByteBuilder::put(const ubyte* arr, size_t size) {
 }
 
 void ByteBuilder::putInt16(int16_t val) {
-    buffer.push_back(static_cast<ubyte>(val >> 0 & 255));
+    size_t size = buffer.size();
-    buffer.push_back(static_cast<ubyte>(val >> 8 & 255));
+    buffer.resize(buffer.size() + sizeof(int16_t));
+    val = dataio::h2le(val);
+    std::memcpy(buffer.data()+size, &val, sizeof(int16_t));
 }
 
 void ByteBuilder::putInt32(int32_t val) {
-    buffer.reserve(buffer.size() + 4);
+    size_t size = buffer.size();
-    buffer.push_back(static_cast<ubyte>(val >> 0 & 255));
+    buffer.resize(buffer.size() + sizeof(int32_t));
-    buffer.push_back(static_cast<ubyte>(val >> 8 & 255));
+    val = dataio::h2le(val);
-    buffer.push_back(static_cast<ubyte>(val >> 16 & 255));
+    std::memcpy(buffer.data()+size, &val, sizeof(int32_t));
-    buffer.push_back(static_cast<ubyte>(val >> 24 & 255));
 }
 
 void ByteBuilder::putInt64(int64_t val) {
-    buffer.reserve(buffer.size() + 8);
+    size_t size = buffer.size();
-    buffer.push_back(static_cast<ubyte>(val >> 0 & 255));
+    buffer.resize(buffer.size() + sizeof(int64_t));
-    buffer.push_back(static_cast<ubyte>(val >> 8 & 255));
+    val = dataio::h2le(val);
-    buffer.push_back(static_cast<ubyte>(val >> 16 & 255));
+    std::memcpy(buffer.data()+size, &val, sizeof(int64_t));
-    buffer.push_back(static_cast<ubyte>(val >> 24 & 255));
-    buffer.push_back(static_cast<ubyte>(val >> 32 & 255));
-    buffer.push_back(static_cast<ubyte>(val >> 40 & 255));
-    buffer.push_back(static_cast<ubyte>(val >> 48 & 255));
-    buffer.push_back(static_cast<ubyte>(val >> 56 & 255));
 }
 
 void ByteBuilder::putFloat32(float val) {
@@ -71,27 +69,18 @@ void ByteBuilder::set(size_t position, ubyte val) {
 }
 
 void ByteBuilder::setInt16(size_t position, int16_t val) {
-    buffer[position++] = val >> 0 & 255;
+    val = dataio::h2le(val);
-    buffer[position] = val >> 8 & 255;
+    std::memcpy(buffer.data()+position, &val, sizeof(int16_t));
 }
 
 void ByteBuilder::setInt32(size_t position, int32_t val) {
-    buffer[position++] = val >> 0 & 255;
+    val = dataio::h2le(val);
-    buffer[position++] = val >> 8 & 255;
+    std::memcpy(buffer.data()+position, &val, sizeof(int32_t));
-    buffer[position++] = val >> 16 & 255;
-    buffer[position] = val >> 24 & 255;
 }
 
 void ByteBuilder::setInt64(size_t position, int64_t val) {
-    buffer[position++] = val >> 0 & 255;
+    val = dataio::h2le(val);
-    buffer[position++] = val >> 8 & 255;
+    std::memcpy(buffer.data()+position, &val, sizeof(int64_t));
-    buffer[position++] = val >> 16 & 255;
-    buffer[position++] = val >> 24 & 255;
-
-    buffer[position++] = val >> 32 & 255;
-    buffer[position++] = val >> 40 & 255;
-    buffer[position++] = val >> 48 & 255;
-    buffer[position] = val >> 56 & 255;
 }
 
 std::vector<ubyte> ByteBuilder::build() {
@@ -111,7 +100,7 @@ void ByteReader::checkMagic(const char* data, size_t size) {
         throw std::runtime_error("invalid magic number");
     }
     for (size_t i = 0; i < size; i++) {
-        if (this->data[pos + i] != (ubyte)data[i]) {
+        if (this->data[pos + i] != static_cast<ubyte>(data[i])) {
             throw std::runtime_error("invalid magic number");
         }
     }
@@ -133,38 +122,33 @@ ubyte ByteReader::peek() {
 }
 
 int16_t ByteReader::getInt16() {
-    if (pos + 2 > size) {
+    if (pos + sizeof(int16_t) > size) {
         throw std::runtime_error("buffer underflow");
     }
-    pos += 2;
+    int16_t value;
-    return (static_cast<int16_t>(data[pos - 1]) << 8) |
+    std::memcpy(&value, data + pos, sizeof(int16_t));
-           (static_cast<int16_t>(data[pos - 2]));
+    pos += sizeof(int16_t);
+    return dataio::le2h(value);
 }
 
 int32_t ByteReader::getInt32() {
-    if (pos + 4 > size) {
+    if (pos + sizeof(int32_t) > size) {
         throw std::runtime_error("buffer underflow");
     }
-    pos += 4;
+    int32_t value;
-    return (static_cast<int32_t>(data[pos - 1]) << 24) |
+    std::memcpy(&value, data + pos, sizeof(int32_t));
-           (static_cast<int32_t>(data[pos - 2]) << 16) |
+    pos += sizeof(int32_t);
-           (static_cast<int32_t>(data[pos - 3]) << 8) |
+    return dataio::le2h(value);
-           (static_cast<int32_t>(data[pos - 4]));
 }
 
 int64_t ByteReader::getInt64() {
-    if (pos + 8 > size) {
+    if (pos + sizeof(int64_t) > size) {
         throw std::runtime_error("buffer underflow");
     }
-    pos += 8;
+    int64_t value;
-    return (static_cast<int64_t>(data[pos - 1]) << 56) |
+    std::memcpy(&value, data + pos, sizeof(int64_t));
-           (static_cast<int64_t>(data[pos - 2]) << 48) |
+    pos += sizeof(int64_t);
-           (static_cast<int64_t>(data[pos - 3]) << 40) |
+    return dataio::le2h(value);
-           (static_cast<int64_t>(data[pos - 4]) << 32) |
-           (static_cast<int64_t>(data[pos - 5]) << 24) |
-           (static_cast<int64_t>(data[pos - 6]) << 16) |
-           (static_cast<int64_t>(data[pos - 7]) << 8) |
-           (static_cast<int64_t>(data[pos - 8]));
 }
 
 float ByteReader::getFloat32() {
@@ -183,7 +167,7 @@ double ByteReader::getFloat64() {
 
 const char* ByteReader::getCString() {
     const char* cstr = reinterpret_cast<const char*>(data + pos);
-    pos += strlen(cstr) + 1;
+    pos += std::strlen(cstr) + 1;
     return cstr;
 }
 

src/coders/byte_utils.hpp

@@ -5,28 +5,27 @@
 
 #include "typedefs.hpp"
 
-/* byteorder: little-endian */
 class ByteBuilder {
     std::vector<ubyte> buffer;
 public:
-    /* Write one byte (8 bit unsigned integer) */
+    /// @brief Write one byte (8 bit unsigned integer)
     void put(ubyte b);
-    /* Write c-string (bytes array terminated with '\00') */
+    /// @brief Write c-string (bytes array terminated with '\00')
     void putCStr(const char* str);
-    /* Write signed 16 bit integer */
+    /// @brief Write signed 16 bit little-endian integer
     void putInt16(int16_t val);
-    /* Write signed 32 bit integer */
+    /// @brief Write signed 32 bit integer
     void putInt32(int32_t val);
-    /* Write signed 64 bit integer */
+    /// @brief Write signed 64 bit integer
     void putInt64(int64_t val);
-    /* Write 32 bit floating-point number */
+    /// @brief Write 32 bit floating-point number
     void putFloat32(float val);
-    /* Write 64 bit floating-point number */
+    /// @brief Write 64 bit floating-point number
     void putFloat64(double val);
 
-    /* Write string (uint32 length + bytes) */
+    /// @brief Write string (uint32 length + bytes)
     void put(const std::string& s);
-    /* Write sequence of bytes without any header */
+    /// @brief Write sequence of bytes without any header
     void put(const ubyte* arr, size_t size);
 
     void set(size_t position, ubyte val);
@@ -44,7 +43,6 @@ public:
     std::vector<ubyte> build();
 };
 
-/// byteorder: little-endian
 class ByteReader {
     const ubyte* data;
     size_t size;
@@ -58,11 +56,11 @@ public:
     ubyte get();
     /// @brief Read one byte (unsigned 8 bit integer) without pointer move
     ubyte peek();
-    /// @brief Read signed 16 bit integer
+    /// @brief Read signed 16 bit little-endian integer
     int16_t getInt16();
-    /// @brief Read signed 32 bit integer
+    /// @brief Read signed 32 bit little-endian integer
     int32_t getInt32();
-    /// @brief Read signed 64 bit integer
+    /// @brief Read signed 64 bit little-endian integer
     int64_t getInt64();
     /// @brief Read 32 bit floating-point number
     float getFloat32();

src/coders/compression.cpp

@@ -0,0 +1,107 @@
+#include "compression.hpp"
+
+#include <string>
+#include <cstring>
+#include <stdexcept>
+
+#include "rle.hpp"
+#include "gzip.hpp"
+#include "util/BufferPool.hpp"
+
+using namespace compression;
+
+static util::BufferPool<ubyte> buffer_pools[] {
+    {255},
+    {UINT16_MAX},
+    {UINT16_MAX * 8},
+};
+
+static std::shared_ptr<ubyte[]> get_buffer(size_t minSize) {
+    for (auto& pool : buffer_pools) {
+        if (minSize <= pool.getBufferSize()) {
+            return pool.get();
+        }
+    }
+    return nullptr;
+}
+
+static auto compress_rle(
+    const ubyte* src,
+    size_t srclen,
+    size_t& len,
+    size_t(*encodefunc)(const ubyte*, size_t, ubyte*)
+) {
+    auto buffer = get_buffer(srclen * 2);
+    auto bytes = buffer.get();
+    std::unique_ptr<ubyte[]> uptr;
+    if (bytes == nullptr) {
+        uptr = std::make_unique<ubyte[]>(srclen * 2);
+        bytes = uptr.get();
+    }
+    len = encodefunc(src, srclen, bytes);
+    if (uptr) {
+        return uptr;
+    }
+    auto data = std::make_unique<ubyte[]>(len);
+    std::memcpy(data.get(), bytes, len);
+    return data;
+}
+
+std::unique_ptr<ubyte[]> compression::compress(
+    const ubyte* src, size_t srclen, size_t& len, Method method
+) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8:
+            return compress_rle(src, srclen, len, extrle::encode);
+        case Method::EXTRLE16:
+            return compress_rle(src, srclen, len, extrle::encode16);
+        case Method::GZIP: {
+            auto buffer = gzip::compress(src, srclen);
+            auto data = std::make_unique<ubyte[]>(buffer.size());
+            std::memcpy(data.get(), buffer.data(), buffer.size());
+            len = buffer.size();
+            return data;
+        }
+        default:
+            throw std::runtime_error("not implemented");
+    }
+}
+
+std::unique_ptr<ubyte[]> compression::decompress(
+    const ubyte* src, size_t srclen, size_t dstlen, Method method
+) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8: {
+            auto decompressed = std::make_unique<ubyte[]>(dstlen);
+            extrle::decode(src, srclen, decompressed.get());
+            return decompressed;
+        }
+        case Method::EXTRLE16: {
+            auto decompressed = std::make_unique<ubyte[]>(dstlen);
+            size_t decoded = extrle::decode16(src, srclen, decompressed.get());
+            if (decoded != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(decoded));
+            }
+            return decompressed;
+        }
+        case Method::GZIP: {
+            auto buffer = gzip::decompress(src, srclen);
+            if (buffer.size() != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(buffer.size()));
+            }
+            auto decompressed = std::make_unique<ubyte[]>(buffer.size());
+            std::memcpy(decompressed.get(), buffer.data(), buffer.size());
+            return decompressed;
+        }
+        default:
+            throw std::runtime_error("not implemented");
+    }
+}

src/coders/compression.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <memory>
+
+#include "typedefs.hpp"
+
+namespace compression {
+    enum class Method {
+        NONE, EXTRLE8, EXTRLE16, GZIP
+    };
+
+    /// @brief Compress buffer
+    /// @param src source buffer
+    /// @param srclen length of the source buffer
+    /// @param len (out argument) length of result buffer
+    /// @param method compression method
+    /// @return compressed bytes array
+    /// @throws std::invalid_argument if compression method is NONE
+    std::unique_ptr<ubyte[]> compress(
+        const ubyte* src, size_t srclen, size_t& len, Method method
+    );
+
+    /// @brief Decompress buffer
+    /// @param src compressed buffer
+    /// @param srclen length of compressed buffer
+    /// @param dstlen max expected length of source buffer
+    /// @return decompressed bytes array
+    std::unique_ptr<ubyte[]> decompress(
+        const ubyte* src, size_t srclen, size_t dstlen, Method method);
+}

src/coders/rle.cpp

@@ -1,5 +1,7 @@
 #include "rle.hpp"
 
+#include "util/data_io.hpp"
+
 size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
@@ -35,13 +37,52 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset;
 }
 
+size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) {
+    auto src16 = reinterpret_cast<const uint16_t*>(src);
+    auto dst16 = reinterpret_cast<uint16_t*>(dst);
+    size_t offset = 0;
+    for (size_t i = 0; i < srclen / 2;) {
+        uint16_t len = dataio::le2h(src16[i++]);
+        uint16_t c = dataio::le2h(src16[i++]);
+        for (size_t j = 0; j <= len; j++) {
+            dst16[offset++] = c;
+        }
+    }
+    return offset * 2;
+}
+
+size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) {
+    if (srclen == 0) {
+        return 0;
+    }
+    auto src16 = reinterpret_cast<const uint16_t*>(src);
+    auto dst16 = reinterpret_cast<uint16_t*>(dst);
+    size_t offset = 0;
+    uint16_t counter = 0;
+    uint16_t c = src16[0];
+    for (size_t i = 1; i < srclen / 2; i++) {
+        uint16_t cnext = src16[i];
+        if (cnext != c || counter == 0xFFFF) {
+            dst16[offset++] = dataio::h2le(counter);
+            dst16[offset++] = dataio::h2le(c);
+            c = cnext;
+            counter = 0;
+        } else {
+            counter++;
+        }
+    }
+    dst16[offset++] = dataio::h2le(counter);
+    dst16[offset++] = dataio::h2le(c);
+    return offset * 2;
+}
+
 size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
         uint len = src[i++];
         if (len & 0x80) {
             len &= 0x7F;
-            len |= ((uint)src[i++]) << 7;
+            len |= (static_cast<uint>(src[i++])) << 7;
         }
         ubyte c = src[i++];
         for (size_t j = 0; j <= len; j++) {
@@ -83,3 +124,70 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     dst[offset++] = c;
     return offset;
 }
+
+size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) {
+    auto dst = reinterpret_cast<uint16_t*>(dst8);
+    size_t offset = 0;
+    for (size_t i = 0; i < srclen;) {
+        uint len = src[i++];
+        bool widechar = len & 0x40;
+        if (len & 0x80) {
+            len &= 0x3F;
+            len |= (static_cast<uint>(src[i++])) << 6;
+        } else {
+            len &= 0x3F;
+        }
+        uint16_t c = src[i++];
+        if (widechar) {
+            c |= ((static_cast<uint>(src[i++])) << 8);
+        }
+        for (size_t j = 0; j <= len; j++) {
+            dst[offset++] = c;
+        }
+    }
+    return offset * 2;
+}
+
+size_t extrle::encode16(const ubyte* src8, size_t srclen, ubyte* dst) {
+    if (srclen == 0) {
+        return 0;
+    }
+    auto src = reinterpret_cast<const uint16_t*>(src8);
+    size_t offset = 0;
+    uint counter = 0;
+    uint16_t c = src[0];
+    for (size_t i = 1; i < srclen/2; i++) {
+        uint16_t cnext = src[i];
+        if (cnext != c || counter == max_sequence16) {
+            if (counter >= 0x40) {
+                dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F);
+                dst[offset++] = counter >> 6;
+            } else {
+                dst[offset++] = counter | ((c > 255) << 6);
+            }
+            if (c > 255) {
+                dst[offset++] = c & 0xFF;
+                dst[offset++] = c >> 8;
+            } else {
+                dst[offset++] = c;
+            }
+            c = cnext;
+            counter = 0;
+        } else {
+            counter++;
+        }
+    }
+    if (counter >= 0x40) {
+        dst[offset++] = 0x80 | ((c > 255) << 6) | (counter & 0x3F);
+        dst[offset++] = counter >> 6;
+    } else {
+        dst[offset++] = counter | ((c > 255) << 6);
+    }
+    if (c > 255) {
+        dst[offset++] = c & 0xFF;
+        dst[offset++] = c >> 8;
+    } else {
+        dst[offset++] = c;
+    }
+    return offset;
+}

src/coders/rle.hpp

@@ -5,10 +5,17 @@
 namespace rle {
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
     size_t decode(const ubyte* src, size_t length, ubyte* dst);
+
+    size_t encode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
 }
 
 namespace extrle {
     constexpr uint max_sequence = 0x7FFF;
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
     size_t decode(const ubyte* src, size_t length, ubyte* dst);
+
+    constexpr uint max_sequence16 = 0x3FFF;
+    size_t encode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
 }

src/constants.hpp

@@ -16,6 +16,12 @@ inline constexpr bool ENGINE_DEBUG_BUILD = true;
 
 inline const std::string ENGINE_VERSION_STRING = "0.23";
 
+/// @brief world regions format version
+inline constexpr uint REGION_FORMAT_VERSION = 3;
+
+/// @brief max simultaneously open world region files
+inline constexpr uint MAX_OPEN_REGION_FILES = 32;
+
 inline constexpr blockid_t BLOCK_AIR = 0;
 inline constexpr blockid_t BLOCK_OBSTACLE = 1;
 inline constexpr itemid_t ITEM_EMPTY = 0;
@@ -41,6 +47,7 @@ inline constexpr itemid_t ITEM_VOID = std::numeric_limits<itemid_t>::max();
 /// @brief max number of block definitions possible
 inline constexpr blockid_t MAX_BLOCKS = BLOCK_VOID;
 
+/// @brief calculates a 1D array index from 3D array indices
 inline constexpr uint vox_index(uint x, uint y, uint z, uint w=CHUNK_W, uint d=CHUNK_D) {
     return (y * d + z) * w + x;
 }

src/content/ContentLUT.cpp

@@ -1,65 +0,0 @@
-#include "ContentLUT.hpp"
-
-#include <memory>
-
-#include "coders/json.hpp"
-#include "constants.hpp"
-#include "files/files.hpp"
-#include "items/ItemDef.hpp"
-#include "voxels/Block.hpp"
-#include "world/World.hpp"
-#include "files/WorldFiles.hpp"
-#include "Content.hpp"
-
-ContentLUT::ContentLUT(
-    const ContentIndices* indices, size_t blocksCount, size_t itemsCount
-)
-    : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK),
-      items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM) {
-}
-
-template <class T>
-static constexpr size_t get_entries_count(
-    const ContentUnitIndices<T>& indices, const dv::value& list
-) {
-    return list != nullptr 
-        ? std::max(list.size(), indices.count()) 
-        : indices.count();
-}
-
-std::shared_ptr<ContentLUT> ContentLUT::create(
-    const std::shared_ptr<WorldFiles>& worldFiles,
-    const fs::path& filename, 
-    const Content* content
-) {
-    auto worldInfo = worldFiles->readWorldInfo();
-    if (!worldInfo.has_value()) {
-        return nullptr;
-    }
-
-    auto root = files::read_json(filename);
-    auto& blocklist = root["blocks"];
-    auto& itemlist = root["items"];
-
-    auto* indices = content->getIndices();
-    size_t blocks_c = get_entries_count(indices->blocks, blocklist);
-    size_t items_c = get_entries_count(indices->items, itemlist);
-
-    auto lut = std::make_shared<ContentLUT>(indices, blocks_c, items_c);
-
-    lut->blocks.setup(blocklist, content->blocks);
-    lut->items.setup(itemlist, content->items);
-
-    if (lut->hasContentReorder() || lut->hasMissingContent()) {
-        return lut;
-    } else {
-        return nullptr;
-    }
-}
-
-std::vector<contententry> ContentLUT::getMissingContent() const {
-    std::vector<contententry> entries;
-    blocks.getMissingContent(entries);
-    items.getMissingContent(entries);
-    return entries;
-}

src/content/ContentLoader.cpp

@@ -21,8 +21,10 @@
 #include "util/stringutil.hpp"
 #include "voxels/Block.hpp"
 #include "data/dv_util.hpp"
+#include "data/StructLayout.hpp"
 
 namespace fs = std::filesystem;
+using namespace data;
 
 static debug::Logger logger("content-loader");
 
@@ -125,6 +127,30 @@ void ContentLoader::fixPackIndices() {
     }
 }
 
+static void perform_user_block_fields(
+    const std::string& blockName, StructLayout& layout
+) {
+    if (layout.size() > MAX_USER_BLOCK_FIELDS_SIZE) {
+        throw std::runtime_error(
+            util::quote(blockName) + 
+            " fields total size exceeds limit (" + 
+            std::to_string(layout.size()) + "/" +
+            std::to_string(MAX_USER_BLOCK_FIELDS_SIZE) + ")");
+    }
+    for (const auto& field : layout) {
+        if (field.name.at(0) == '.') {
+            throw std::runtime_error(
+                util::quote(blockName) + " field " + field.name + 
+                ": user field may not start with '.'");
+        }
+    }
+
+    std::vector<Field> fields;
+    fields.insert(fields.end(), layout.begin(), layout.end());
+    // add built-in fields here
+    layout = StructLayout::create(fields);
+}
+
 void ContentLoader::loadBlock(
     Block& def, const std::string& name, const fs::path& file
 ) {
@@ -259,6 +285,14 @@ void ContentLoader::loadBlock(
     root.at("ui-layout").get(def.uiLayout);
     root.at("inventory-size").get(def.inventorySize);
     root.at("tick-interval").get(def.tickInterval);
+
+    if (root.has("fields")) {
+        def.dataStruct = std::make_unique<StructLayout>();
+        def.dataStruct->deserialize(root["fields"]);
+
+        perform_user_block_fields(def.name, *def.dataStruct);
+    }
+
     if (def.tickInterval == 0) {
         def.tickInterval = 1;
     }

src/content/ContentReport.cpp

@@ -0,0 +1,146 @@
+#include "ContentReport.hpp"
+
+#include <memory>
+
+#include "coders/json.hpp"
+#include "constants.hpp"
+#include "files/files.hpp"
+#include "items/ItemDef.hpp"
+#include "voxels/Block.hpp"
+#include "world/World.hpp"
+#include "files/WorldFiles.hpp"
+#include "Content.hpp"
+
+ContentReport::ContentReport(
+    const ContentIndices* indices,
+    size_t blocksCount,
+    size_t itemsCount,
+    uint regionsVersion
+)
+    : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK),
+      items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM),
+      regionsVersion(regionsVersion) {
+}
+
+template <class T>
+static constexpr size_t get_entries_count(
+    const ContentUnitIndices<T>& indices, const dv::value& list
+) {
+    return list != nullptr ? std::max(list.size(), indices.count())
+                           : indices.count();
+}
+
+static void process_blocks_data(
+    const Content* content, ContentReport& report, const dv::value& root
+) {
+    for (const auto& [name, map] : root.asObject()) {
+        data::StructLayout layout;
+        layout.deserialize(map);
+        auto def = content->blocks.find(name);
+        if (def == nullptr) {
+            continue;
+        }
+        if (def->dataStruct == nullptr) {
+            ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
+            report.issues.push_back(issue);
+            report.dataLoss.push_back(name + ": discard data");
+            continue;
+        }
+        if (layout != *def->dataStruct) {
+            ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
+            report.issues.push_back(issue);
+            report.dataLayoutsUpdated = true;
+        }
+
+        auto incapatibility = layout.checkCompatibility(*def->dataStruct);
+        if (!incapatibility.empty()) {
+            for (const auto& error : incapatibility) {
+                report.dataLoss.push_back(
+                    "[" + name + "] field " + error.name + " - " +
+                    data::to_string(error.type)
+                );
+            }
+        }
+        report.blocksDataLayouts[name] = std::move(layout);
+    }
+}
+
+std::shared_ptr<ContentReport> ContentReport::create(
+    const std::shared_ptr<WorldFiles>& worldFiles,
+    const fs::path& filename,
+    const Content* content
+) {
+    auto worldInfo = worldFiles->readWorldInfo();
+    if (!worldInfo.has_value()) {
+        return nullptr;
+    }
+
+    auto root = files::read_json(filename);
+    // TODO: remove default value 2 in 0.24
+    uint regionsVersion = 2U;
+    root.at("region-version").get(regionsVersion);
+    auto& blocklist = root["blocks"];
+    auto& itemlist = root["items"];
+
+    auto* indices = content->getIndices();
+    size_t blocks_c = get_entries_count(indices->blocks, blocklist);
+    size_t items_c = get_entries_count(indices->items, itemlist);
+
+    auto report = std::make_shared<ContentReport>(
+        indices, blocks_c, items_c, regionsVersion
+    );
+    report->blocks.setup(blocklist, content->blocks);
+    report->items.setup(itemlist, content->items);
+
+    if (root.has("blocks-data")) {
+        process_blocks_data(content, *report, root["blocks-data"]);
+    }
+
+    report->buildIssues();
+
+    if (report->isUpgradeRequired() || report->hasContentReorder() ||
+        report->hasMissingContent() || report->hasUpdatedLayouts()) {
+        return report;
+    } else {
+        return nullptr;
+    }
+}
+
+template <class T, class U>
+static void build_issues(
+    std::vector<ContentIssue>& issues, const ContentUnitLUT<T, U>& report
+) {
+    auto type = report.getContentType();
+    if (report.hasContentReorder()) {
+        issues.push_back(ContentIssue {ContentIssueType::REORDER, type});
+    }
+    if (report.hasMissingContent()) {
+        issues.push_back(ContentIssue {ContentIssueType::MISSING, type});
+    }
+}
+
+void ContentReport::buildIssues() {
+    build_issues(issues, blocks);
+    build_issues(issues, items);
+    
+    if (regionsVersion < REGION_FORMAT_VERSION) {
+        for (int layer = REGION_LAYER_VOXELS; 
+             layer < REGION_LAYERS_COUNT; 
+             layer++) {
+            ContentIssue issue {ContentIssueType::REGION_FORMAT_UPDATE};
+            issue.regionLayer = static_cast<RegionLayerIndex>(layer);
+            issues.push_back(issue);
+        }
+    }
+}
+
+const std::vector<ContentIssue>& ContentReport::getIssues() const {
+    return issues;
+}
+
+std::vector<ContentEntry> ContentReport::getMissingContent() const {
+    std::vector<ContentEntry> entries;
+    blocks.getMissingContent(entries);
+    items.getMissingContent(entries);
+    return entries;
+}

src/content/ContentReport.hpp

@@ -4,27 +4,49 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <unordered_map>
 
 #include "constants.hpp"
 #include "data/dv.hpp"
 #include "typedefs.hpp"
 #include "Content.hpp"
+#include "data/StructLayout.hpp"
+#include "files/world_regions_fwd.hpp"
 
 namespace fs = std::filesystem;
 
-struct contententry {
+enum class ContentIssueType {
+    REORDER,
+    MISSING,
+    REGION_FORMAT_UPDATE,
+    BLOCK_DATA_LAYOUTS_UPDATE,
+};
+
+struct ContentIssue {
+    ContentIssueType issueType;
+    union {
+        ContentType contentType;
+        RegionLayerIndex regionLayer;
+    };
+};
+
+struct ContentEntry {
     ContentType type;
     std::string name;
 };
 
 class WorldFiles;
 
+/// @brief Content unit lookup table
+/// @tparam T index type
+/// @tparam U unit class
 template <typename T, class U>
 class ContentUnitLUT {
     std::vector<T> indices;
     std::vector<std::string> names;
     bool missingContent = false;
     bool reorderContent = false;
+    /// @brief index that will be used to mark missing unit
     T missingValue;
     ContentType type;
 public:
@@ -57,11 +79,11 @@ public:
             }
         }
     }
-    void getMissingContent(std::vector<contententry>& entries) const {
+    void getMissingContent(std::vector<ContentEntry>& entries) const {
         for (size_t i = 0; i < count(); i++) {
             if (indices[i] == missingValue) {
                 auto& name = names[i];
-                entries.push_back(contententry {type, name});
+                entries.push_back(ContentEntry {type, name});
             }
         }
     }
@@ -80,6 +102,9 @@ public:
             reorderContent = true;
         }
     }
+    inline ContentType getContentType() const {
+        return type;
+    }
     inline size_t count() const {
         return indices.size();
     }
@@ -91,28 +116,54 @@ public:
     }
 };
 
-/// @brief Content indices lookup table or report
+/// @brief Content incapatibility report used to convert world.
-/// used to convert world with different indices
 /// Building with indices.json
-class ContentLUT {
+class ContentReport {
 public:
     ContentUnitLUT<blockid_t, Block> blocks;
     ContentUnitLUT<itemid_t, ItemDef> items;
+    uint regionsVersion;
 
-    ContentLUT(const ContentIndices* indices, size_t blocks, size_t items);
+    std::unordered_map<std::string, data::StructLayout> blocksDataLayouts;
+    std::vector<ContentIssue> issues;
+    std::vector<std::string> dataLoss;
 
-    static std::shared_ptr<ContentLUT> create(
+    bool dataLayoutsUpdated = false;
+
+    ContentReport(
+        const ContentIndices* indices, 
+        size_t blocks, 
+        size_t items,
+        uint regionsVersion
+    );
+
+    static std::shared_ptr<ContentReport> create(
         const std::shared_ptr<WorldFiles>& worldFiles,
         const fs::path& filename,
         const Content* content
     );
 
+    inline const std::vector<std::string>& getDataLoss() const {
+        return dataLoss;
+    }
+    inline bool hasUpdatedLayouts() {
+        return dataLayoutsUpdated;
+    }
+
     inline bool hasContentReorder() const {
         return blocks.hasContentReorder() || items.hasContentReorder();
     }
     inline bool hasMissingContent() const {
         return blocks.hasMissingContent() || items.hasMissingContent();
     }
+    inline bool isUpgradeRequired() const {
+        return regionsVersion < REGION_FORMAT_VERSION;
+    }
+    inline bool hasDataLoss() const {
+        return !dataLoss.empty();
+    }
+    void buildIssues();
 
-    std::vector<contententry> getMissingContent() const;
+    const std::vector<ContentIssue>& getIssues() const;
+    std::vector<ContentEntry> getMissingContent() const;
 };

src/data/StructLayout.cpp

@@ -0,0 +1,391 @@
+#include "StructLayout.hpp"
+
+#include <map>
+#include <cstring>
+#include <climits>
+#include <string.h>
+#include <algorithm>
+
+#include "util/data_io.hpp"
+#include "util/stringutil.hpp"
+
+using namespace data;
+
+static_assert(sizeof(float) == sizeof(int32_t));
+static_assert(sizeof(double) == sizeof(int64_t));
+
+FieldType data::FieldType_from_string(std::string_view name) {
+    std::map<std::string_view, FieldType> map {
+        {"int8", FieldType::I8},
+        {"int16", FieldType::I16},
+        {"int32", FieldType::I32},
+        {"int64", FieldType::I64},
+        {"float32", FieldType::F32},
+        {"float64", FieldType::F64},
+        {"char", FieldType::CHAR},
+    };
+    return map.at(name);
+}
+
+FieldConvertStrategy data::FieldConvertStrategy_from_string(std::string_view name) {
+    std::map<std::string_view, FieldConvertStrategy> map {
+        {"reset", FieldConvertStrategy::RESET},
+        {"clamp", FieldConvertStrategy::CLAMP}
+    };
+    return map.at(name);
+}
+
+StructLayout StructLayout::create(const std::vector<Field>& fields) {
+    std::vector<Field> builtFields = fields;
+    std::unordered_map<std::string, int> indices;
+
+    for (Field& field : builtFields) {
+        field.size = sizeof_type(field.type) * field.elements;
+    }
+    std::sort(builtFields.begin(), builtFields.end(), 
+        [](const Field& a, const Field& b) {
+            return a.size > b.size;
+        }
+    );
+    int offset = 0;
+    for (int i = 0; i < builtFields.size(); i++) {
+        auto& field = builtFields[i];
+        field.offset = offset;
+        indices[field.name] = i;
+        offset += field.size;
+    }
+    return StructLayout(
+        offset, std::move(builtFields), std::move(indices));
+}
+
+static inline constexpr bool is_integer_type(FieldType type) {
+    return (type >= FieldType::I8 && type <= FieldType::I64) || 
+            type == FieldType::CHAR;
+}
+
+static inline constexpr bool is_floating_point_type(FieldType type) {
+    return type == FieldType::F32 || type == FieldType::F64;
+}
+
+static inline constexpr bool is_numeric_type(FieldType type) {
+    return is_floating_point_type(type) || is_integer_type(type);
+}
+
+static inline FieldIncapatibilityType checkIncapatibility(
+    const Field& srcField, const Field& dstField
+) {
+    auto type = FieldIncapatibilityType::NONE;
+    if (dstField.elements < srcField.elements) {
+        type = FieldIncapatibilityType::DATA_LOSS;
+    }
+    if (srcField.type == dstField.type) {
+        return type;
+    }
+    if (is_numeric_type(srcField.type) && is_numeric_type(dstField.type)) {
+        int sizediff =
+            sizeof_type(dstField.type) - sizeof_type(srcField.type);
+        if (sizediff < 0) {
+            type = std::max(type, FieldIncapatibilityType::DATA_LOSS);
+        }
+    } else {
+        type = std::max(type, FieldIncapatibilityType::TYPE_ERROR);
+    }
+    return type;
+}
+
+static inline integer_t clamp_value(integer_t value, FieldType type) {
+    auto typesize = sizeof_type(type) * CHAR_BIT;
+    integer_t minval = -(1 << (typesize-1));
+    integer_t maxval = (1 << (typesize-1))-1;
+    return std::min(maxval, std::max(minval, value));
+}
+
+static void reset_integer(
+    const StructLayout& srcLayout,
+    const StructLayout& dstLayout,
+    const Field& field,
+    const Field& dstField,
+    const ubyte* src, 
+    ubyte* dst
+) {
+    int elements = std::min(field.elements, dstField.elements);
+    for (int i = 0; i < elements; i++) {
+        auto value = srcLayout.getInteger(src, field.name, i);
+        auto clamped = clamp_value(value, dstField.type);
+        if (dstField.convertStrategy == FieldConvertStrategy::CLAMP) {
+            value = clamped;
+        } else {
+            if (clamped != value) {
+                value = 0;
+            }
+        }
+        dstLayout.setInteger(dst, value, field.name, i);
+    }
+}
+
+static void reset_number(
+    const StructLayout& srcLayout,
+    const StructLayout& dstLayout,
+    const Field& field,
+    const Field& dstField,
+    const ubyte* src, 
+    ubyte* dst
+) {
+    int elements = std::min(field.elements, dstField.elements);
+    for (int i = 0; i < elements; i++) {
+        auto value = srcLayout.getNumber(src, field.name, i);
+        dstLayout.setNumber(dst, value, field.name, i);
+    }
+}
+
+void StructLayout::convert(
+    const StructLayout& srcLayout, 
+    const ubyte* src, 
+    ubyte* dst,
+    bool allowDataLoss
+) const {
+    std::memset(dst, 0, totalSize);
+    for (const Field& field : srcLayout.fields) {
+        auto dstField = getField(field.name);
+        if (dstField == nullptr) {
+            continue;
+        }
+        auto type = checkIncapatibility(field, *dstField);
+        if (type == FieldIncapatibilityType::TYPE_ERROR) {
+            continue;
+        }
+        // can't just memcpy, because field type may be changed without data loss
+        if (is_integer_type(field.type) ||
+                (is_floating_point_type(field.type) && 
+                    is_integer_type(dstField->type))) {
+            reset_integer(srcLayout, *this, field, *dstField, src, dst);
+        } else if (is_floating_point_type(dstField->type)) {
+            reset_number(srcLayout, *this, field, *dstField, src, dst);
+        }
+    }
+}
+
+std::vector<FieldIncapatibility> StructLayout::checkCompatibility(
+    const StructLayout& dstLayout
+) {
+    std::vector<FieldIncapatibility> report;
+    for (const Field& field : fields) {
+        auto dstField = dstLayout.getField(field.name);
+        if (dstField == nullptr) {
+            report.push_back({field.name, FieldIncapatibilityType::MISSING});
+            continue;
+        }
+        auto type = checkIncapatibility(field, *dstField);
+        if (type != FieldIncapatibilityType::NONE) {
+            report.push_back({field.name, type});
+        }
+    }
+    return report;
+}
+
+const Field& StructLayout::requireField(const std::string& name) const {
+    auto found = indices.find(name);
+    if (found == indices.end()) {
+        throw std::runtime_error("field '"+name+"' does not exist");
+    }
+    return *&fields.at(found->second);
+}
+
+
+template<typename T>
+static void set_int(ubyte* dst, integer_t value) {
+    T out_value = static_cast<T>(value);
+    out_value = dataio::le2h(out_value);
+    *reinterpret_cast<T*>(dst) = out_value;
+}
+
+void StructLayout::setInteger(
+    ubyte* dst, integer_t value, const Field& field, int index
+) const {
+    if (index < 0 || index >= field.elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field.elements)+"]");
+    }
+    auto ptr = dst + field.offset + index * sizeof_type(field.type);
+    switch (field.type) {
+        case FieldType::I8: set_int<int8_t>(ptr, value); break;
+        case FieldType::I16: set_int<int16_t>(ptr, value); break;
+        case FieldType::I32: set_int<int32_t>(ptr, value); break;
+        case FieldType::I64: set_int<int64_t>(ptr, value); break;
+        case FieldType::CHAR: set_int<int8_t>(ptr, value); break;
+        case FieldType::F32:
+        case FieldType::F64:
+            setNumber(dst, static_cast<number_t>(value), field, index);
+            break;
+        default: 
+            throw std::runtime_error("type error");
+    }
+}
+
+void StructLayout::setNumber(
+    ubyte* dst, number_t value, const Field& field, int index
+) const {
+    if (index < 0 || index >= field.elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field.elements)+"]");
+    }
+    auto ptr = dst + field.offset + index * sizeof_type(field.type);
+    switch (field.type) {
+        case FieldType::F32: {
+            float fval = static_cast<float>(value);
+            int32_t ival;
+            std::memcpy(&ival, &fval, sizeof(int32_t));
+            set_int<int32_t>(ptr, ival);
+            break;
+        }
+        case FieldType::F64: {
+            double fval = static_cast<double>(value);
+            int64_t ival;
+            std::memcpy(&ival, &fval, sizeof(int64_t));
+            set_int<int64_t>(ptr, ival);
+            break;
+        }
+        default:
+            throw std::runtime_error("type error");
+    }
+}
+
+size_t StructLayout::setAscii(
+    ubyte* dst, std::string_view value, const std::string& name
+) const {
+    const auto& field = requireField(name);
+    if (field.type != FieldType::CHAR) {
+        throw std::runtime_error("'char' field type required");
+    }
+    auto ptr = reinterpret_cast<char*>(dst + field.offset);
+    auto size = std::min(value.size(), static_cast<std::size_t>(field.elements));
+    std::memcpy(ptr, value.data(), size);
+    if (size < field.elements) {
+        std::memset(ptr + size, 0, field.elements - size);
+    }
+    return size;
+}
+
+size_t StructLayout::setUnicode(
+    ubyte* dst, std::string_view value, const Field& field
+) const {
+    if (field.type != FieldType::CHAR) {
+        throw std::runtime_error("'char' field type required");
+    }
+    auto text = std::string_view(value.data(), value.size());
+    size_t size = util::crop_utf8(text, field.elements);
+    auto ptr = reinterpret_cast<char*>(dst + field.offset);
+    std::memcpy(ptr, value.data(), size);
+    if (size < field.elements) {
+        std::memset(ptr + size, 0, field.elements - size);
+    }
+    return size;
+}
+
+template<typename T>
+static T get_int(const ubyte* src) {
+    return dataio::le2h(*reinterpret_cast<const T*>(src));
+}
+
+integer_t StructLayout::getInteger(
+    const ubyte* src, const Field& field, int index
+) const {
+    if (index < 0 || index >= field.elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field.elements)+"]");
+    }
+    auto ptr = src + field.offset + index * sizeof_type(field.type);
+    switch (field.type) {
+        case FieldType::I8: return get_int<int8_t>(ptr);
+        case FieldType::I16: return get_int<int16_t>(ptr);
+        case FieldType::I32: return get_int<int32_t>(ptr);
+        case FieldType::I64: return get_int<int64_t>(ptr);
+        case FieldType::CHAR: return get_int<int8_t>(ptr);
+        default:
+            throw std::runtime_error("type error");
+    }
+}
+
+number_t StructLayout::getNumber(
+    const ubyte* src, const Field& field, int index
+) const {
+    if (index < 0 || index >= field.elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field.elements)+"]");
+    }
+    auto ptr = src + field.offset + index * sizeof_type(field.type);
+    switch (field.type) {
+        case FieldType::F32: {
+            float fval;
+            auto ival = get_int<int32_t>(ptr);
+            std::memcpy(&fval, &ival, sizeof(float));
+            return fval;
+        }
+        case FieldType::F64: {
+            double fval;
+            auto ival = get_int<int64_t>(ptr);
+            std::memcpy(&fval, &ival, sizeof(double));
+            return fval;
+        }
+        case FieldType::I8:
+        case FieldType::I16:
+        case FieldType::I32:
+        case FieldType::I64:
+        case FieldType::CHAR:
+            return getInteger(src, field, index);
+        default:
+            throw std::runtime_error("type error");
+    }
+}
+
+std::string_view StructLayout::getChars(
+    const ubyte* src, const Field& field
+) const {
+    if (field.type != FieldType::CHAR) {
+        throw std::runtime_error("'char' field type required");
+    }
+    auto ptr = reinterpret_cast<const char*>(src + field.offset);
+    return std::string_view(ptr, strnlen(ptr, field.elements));
+}
+
+dv::value StructLayout::serialize() const {
+    auto map = dv::object();
+    for (const auto& [name, index] : indices) {
+        auto& fieldmap = map.object(name);
+        const auto& field = fields[index];
+        fieldmap["type"] = to_string(field.type);
+        if (field.elements != 1) {
+            fieldmap["length"] = field.elements;
+        }
+        if (field.convertStrategy != FieldConvertStrategy::RESET) {
+            fieldmap["convert-strategy"] = to_string(field.convertStrategy);
+        }
+    }
+    return map;
+}
+
+void StructLayout::deserialize(const dv::value& src) {
+    std::vector<Field> fields;
+    for (const auto& [name, fieldmap] : src.asObject()) {
+        const auto& typeName = fieldmap["type"].asString();
+        FieldType type = FieldType_from_string(typeName);
+        
+        int elements = 1;
+        fieldmap.at("length").get(elements);
+        if (elements <= 0) {
+            throw std::runtime_error(
+                "invalid field " + util::quote(name) + " length: " +
+                std::to_string(elements)
+            );
+        }
+
+        auto convertStrategy = FieldConvertStrategy::RESET;
+        if (fieldmap.has("convert-strategy")) {
+            convertStrategy = FieldConvertStrategy_from_string(
+                fieldmap["convert-strategy"].asString()
+            );
+        }
+        fields.push_back(Field (type, name, elements, convertStrategy));
+    }
+    *this = create(fields);
+}

src/data/StructLayout.hpp

@@ -0,0 +1,280 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <stdexcept>
+#include <unordered_map>
+#include <optional>
+
+#include "typedefs.hpp"
+#include "interfaces/Serializable.hpp"
+
+namespace data {
+    enum class FieldType {
+        I8=0, I16, I32, I64, F32, F64, CHAR
+    };
+
+    inline std::string to_string(FieldType type) {
+        const char* names[] = {
+            "int8", "int16", "int32", "int64", "float32", "float64", "char" 
+        };
+        return names[static_cast<int>(type)];
+    }
+    FieldType FieldType_from_string(std::string_view name);
+
+    /// @brief Sorted by severity
+    enum class FieldIncapatibilityType {
+        NONE = 0,
+        DATA_LOSS,
+        TYPE_ERROR,
+        MISSING,
+    };
+    inline const char* to_string(FieldIncapatibilityType type) {
+        const char* names[] = {
+            "none", "data_loss", "type_error", "missing"
+        };
+        return names[static_cast<int>(type)];
+    }
+
+    struct FieldIncapatibility {
+        std::string name;
+        FieldIncapatibilityType type;
+    };
+
+    inline constexpr int sizeof_type(FieldType type) {
+        const int sizes[] = {
+            1, 2, 4, 8, 4, 8, 1
+        };
+        return sizes[static_cast<int>(type)];
+    }
+
+    class dataloss_error : public std::runtime_error {
+    public:
+        dataloss_error(const std::string& message) : std::runtime_error(message) {}
+    };
+
+    /// @brief Strategy will be used if value can't be left the same on conversion
+    enum class FieldConvertStrategy {
+        /// @brief Reset field value
+        RESET = 0,
+        /// @brief Clamp field value if out of type range
+        CLAMP
+    };
+    
+    inline const char* to_string(FieldConvertStrategy strategy) {
+        const char* names[] = {
+            "reset", "clamp"
+        };
+        return names[static_cast<int>(strategy)];
+    }
+    FieldConvertStrategy FieldConvertStrategy_from_string(std::string_view name);
+
+    struct Field {
+        FieldType type;
+        std::string name;
+        /// @brief Number of field elements (array size)
+        int elements;
+        /// @brief Strategy will be used in data loss case
+        FieldConvertStrategy convertStrategy;
+        /// @brief Byte offset of the field
+        int offset;
+        /// @brief Byte size of the field
+        int size;
+
+        Field(
+            FieldType type, 
+            std::string name, 
+            int elements, 
+            FieldConvertStrategy strategy=FieldConvertStrategy::RESET
+        ) : type(type), 
+            name(std::move(name)), 
+            elements(elements), 
+            convertStrategy(strategy),
+            offset(0),
+            size(0) {}
+
+        bool operator==(const Field& o) const {
+            return type == o.type && 
+                   name == o.name && 
+                   elements == o.elements &&
+                   convertStrategy == o.convertStrategy &&
+                   offset == o.offset &&
+                   size == o.size;
+        }
+        bool operator!=(const Field& o) const {
+            return !operator==(o);
+        }
+    };
+    
+    class StructLayout : public Serializable {
+        int totalSize;
+        std::vector<Field> fields;
+        std::unordered_map<std::string, int> indices;
+    
+        StructLayout(
+            int totalSize,
+            std::vector<Field> fields,
+            std::unordered_map<std::string, int> indices
+        ) : totalSize(totalSize), 
+            fields(std::move(fields)),
+            indices(std::move(indices))
+        {}
+    public:
+        StructLayout() : StructLayout(0, {}, {}) {}
+
+        bool operator==(const StructLayout& o) const {
+            // if fields are completely equal then structures are equal
+            return fields == o.fields;
+        }
+        bool operator!=(const StructLayout& o) const {
+            return !operator==(o);
+        }
+
+        /// @brief Get field by name. Returns nullptr if field not found.
+        /// @param name field name
+        /// @return nullable field pointer
+        [[nodiscard]]
+        const Field* getField(const std::string& name) const {
+            auto found = indices.find(name);
+            if (found == indices.end()) {
+                return nullptr;
+            }
+            return &fields.at(found->second);
+        }
+
+        /// @brief Get field by name
+        /// @throws std::runtime_exception - field not found
+        /// @param name field name
+        /// @return read-only field reference
+        const Field& requireField(const std::string& name) const;
+
+        [[nodiscard]]
+        integer_t getInteger(const ubyte* src, const std::string& name, int index=0) const {
+            return getInteger(src, requireField(name), index);
+        }
+
+        /// @brief Get integer from specified field. 
+        /// Types: (i8, i16, i32, i64, char)
+        /// @throws std::runtime_exception - field not found
+        /// @throws std::out_of_range - index is out of range [0, elements-1]
+        /// @param src source buffer
+        /// @param field target field
+        /// @param index array index
+        /// @return field value
+        [[nodiscard]]
+        integer_t getInteger(const ubyte* src, const Field& field, int index=0) const;
+
+        [[nodiscard]]
+        number_t getNumber(const ubyte* src, const std::string& name, int index=0) const {
+            return getNumber(src, requireField(name), index);
+        }
+
+        /// @brief Get floating-point number from specified field. 
+        /// Types: (f32, f64, i8, i16, i32, i64, char)
+        /// @throws std::runtime_exception - field not found
+        /// @throws std::out_of_range - index is out of range [0, elements-1]
+        /// @param src source buffer
+        /// @param field target field
+        /// @param index array index
+        /// @return field value
+        [[nodiscard]]
+        number_t getNumber(const ubyte* src, const Field& field, int index=0) const;
+
+        [[nodiscard]]
+        std::string_view getChars(const ubyte* src, const std::string& name) const {
+            return getChars(src, requireField(name));
+        }
+
+        /// @brief Get read-only chars array as string_view.
+        /// @param src source buffer
+        /// @param field target field
+        [[nodiscard]]
+        std::string_view getChars(const ubyte* src, const Field& field) const;
+
+        void setInteger(ubyte* dst, integer_t value, const std::string& name, int index=0) const {
+            setInteger(dst, value, requireField(name), index);
+        }
+
+        /// @brief Set field integer value.
+        /// Types: (i8, i16, i32, i64, f32, f64, char)
+        /// @throws std::runtime_exception - field not found
+        /// @throws std::out_of_range - index is out of range [0, elements-1]
+        /// @param dst destination buffer
+        /// @param value value
+        /// @param field target field
+        /// @param index array index
+        void setInteger(ubyte* dst, integer_t value, const Field& field, int index=0) const;
+
+        void setNumber(ubyte* dst, number_t value, const std::string& name, int index=0) const {
+            setNumber(dst, value, requireField(name), index);
+        }
+
+        /// @brief Set field numeric value.
+        /// Types: (f32, f64)
+        /// @throws std::runtime_exception - field not found
+        /// @throws std::out_of_range - index is out of range [0, elements-1]
+        /// @param dst destination buffer
+        /// @param value value
+        /// @param field target field
+        /// @param index array index
+        void setNumber(ubyte* dst, number_t value, const Field& field, int index=0) const;
+        
+        /// @brief Replace chars array to given ASCII string
+        /// @throws std::runtime_exception - field not found
+        /// @see StructMapper::setUnicode - utf-8 version of setAscii
+        /// @param dst destination buffer
+        /// @param value ASCII string
+        /// @param name field name
+        /// @return number of written string chars
+        size_t setAscii(ubyte* dst, std::string_view value, const std::string& name) const;
+
+        size_t setUnicode(ubyte* dst, std::string_view value, const std::string& name) const {
+            return setUnicode(dst, value, requireField(name));
+        }
+        
+        /// @brief Unicode-safe version of setAscii
+        /// @throws std::runtime_exception - field not found
+        /// @param dst destination buffer
+        /// @param value utf-8 string
+        /// @param name field name
+        /// @return number of written string chars
+        size_t setUnicode(ubyte* dst, std::string_view value, const Field& field) const;
+
+        /// @return total structure size (bytes)
+        [[nodiscard]] size_t size() const {
+            return totalSize;
+        }
+
+        [[nodiscard]] const auto begin() const {
+            return fields.begin();
+        }
+
+        [[nodiscard]] const auto end() const {
+            return fields.end();
+        }
+
+        /// @brief Convert structure data from srcLayout to this layout.
+        /// @param srcLayout source structure layout
+        /// @param src source data
+        /// @param dst destination buffer
+        /// (size must be enough to store converted structure) 
+        /// @param allowDataLoss allow to drop fields that are not present in 
+        /// this layout or have incompatible types
+        /// @throws data::dataloss_error - data loss detected and allowDataLoss 
+        /// is set to false
+        void convert(
+            const StructLayout& srcLayout, 
+            const ubyte* src, 
+            ubyte* dst,
+            bool allowDataLoss) const;
+
+        std::vector<FieldIncapatibility> checkCompatibility(
+            const StructLayout& dstLayout);
+
+        [[nodiscard]]
+        static StructLayout create(const std::vector<Field>& fields);
+
+        dv::value serialize() const override;
+        void deserialize(const dv::value& src) override;
+    };
+}

src/data/dv.hpp

@@ -518,12 +518,16 @@ namespace dv {
         return std::make_shared<objects::Object>();
     }
 
+    inline value object(std::initializer_list<pair> pairs) {
+        return std::make_shared<objects::Object>(std::move(pairs));
+    }
+
     inline value list() {
         return std::make_shared<objects::List>();
     }
 
     inline value list(std::initializer_list<value> values) {
-        return std::make_shared<objects::List>(values);
+        return std::make_shared<objects::List>(std::move(values));
     }
 
     template<typename T> inline bool get_to_int(value* ptr, T& dst) {

src/files/RegionsLayer.cpp

@@ -0,0 +1,241 @@
+#include "WorldRegions.hpp"
+
+#include <cstring>
+
+#include "util/data_io.hpp"
+
+#define REGION_FORMAT_MAGIC ".VOXREG"
+
+static fs::path get_region_filename(int x, int z) {
+    return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
+}
+
+/// @brief Read missing chunks data (null pointers) from region file
+static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) {
+    auto* chunks = region->getChunks();
+    auto sizes = region->getSizes();
+
+    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
+        int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
+        int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
+        if (chunks[i] == nullptr) {
+            chunks[i] = RegionsLayer::readChunkData(
+                    chunk_x, chunk_z, sizes[i][0], sizes[i][1], file);
+        }
+    }
+}
+
+regfile::regfile(fs::path filename) : file(std::move(filename)) {
+    if (file.length() < REGION_HEADER_SIZE)
+        throw std::runtime_error("incomplete region file header");
+    char header[REGION_HEADER_SIZE];
+    file.read(header, REGION_HEADER_SIZE);
+
+    // avoid of use strcmp_s
+    if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) !=
+        REGION_FORMAT_MAGIC) {
+        throw std::runtime_error("invalid region file magic number");
+    }
+    version = header[8];
+    if (static_cast<uint>(version) > REGION_FORMAT_VERSION) {
+        throw illegal_region_format(
+            "region format " + std::to_string(version) + " is not supported"
+        );
+    }
+}
+
+std::unique_ptr<ubyte[]> regfile::read(int index, uint32_t& size, uint32_t& srcSize) {
+    size_t file_size = file.length();
+    size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
+
+    uint32_t buff32;
+    file.seekg(table_offset + index * 4);
+    file.read(reinterpret_cast<char*>(&buff32), 4);
+    uint32_t offset = dataio::le2h(buff32);
+    if (offset == 0) {
+        return nullptr;
+    }
+
+    file.seekg(offset);
+    file.read(reinterpret_cast<char*>(&buff32), 4);
+    size = dataio::le2h(buff32);
+    file.read(reinterpret_cast<char*>(&buff32), 4);
+    srcSize = dataio::le2h(buff32);
+
+    auto data = std::make_unique<ubyte[]>(size);
+    file.read(reinterpret_cast<char*>(data.get()), size);
+    return data;
+}
+
+
+void RegionsLayer::closeRegFile(glm::ivec2 coord) {
+    openRegFiles.erase(coord);
+    regFilesCv.notify_one();
+}
+
+regfile_ptr RegionsLayer::useRegFile(glm::ivec2 coord) {
+    auto* file = openRegFiles[coord].get();
+    file->inUse = true;
+    return regfile_ptr(file, &regFilesCv);
+}
+
+// Marks regfile as used and unmarks when shared_ptr dies
+regfile_ptr RegionsLayer::getRegFile(glm::ivec2 coord, bool create) {
+    {
+        std::lock_guard lock(regFilesMutex);
+        const auto found = openRegFiles.find(coord);
+        if (found != openRegFiles.end()) {
+            if (found->second->inUse) {
+                throw std::runtime_error("regfile is currently in use");
+            }
+            return useRegFile(found->first);
+        }
+    }
+    if (create) {
+        return createRegFile(coord);
+    }
+    return nullptr;
+}
+
+regfile_ptr RegionsLayer::createRegFile(glm::ivec2 coord) {
+    auto file =  folder / get_region_filename(coord[0], coord[1]);
+    if (!fs::exists(file)) {
+        return nullptr;
+    }
+    if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
+        std::unique_lock lock(regFilesMutex);
+        while (true) {
+            bool closed = false;
+            // FIXME: bad choosing algorithm
+            for (auto& entry : openRegFiles) {
+                if (!entry.second->inUse) {
+                    closeRegFile(entry.first);
+                    closed = true;
+                    break;
+                }
+            }
+            if (closed) {
+                break;
+            }
+            // notified when any regfile gets out of use or closed
+            regFilesCv.wait(lock);
+        }
+        openRegFiles[coord] = std::make_unique<regfile>(file);
+        return useRegFile(coord);
+    } else {
+        std::lock_guard lock(regFilesMutex);
+        openRegFiles[coord] = std::make_unique<regfile>(file);
+        return useRegFile(coord);
+    }
+}
+
+WorldRegion* RegionsLayer::getRegion(int x, int z) {
+    std::lock_guard lock(mapMutex);
+    auto found = regions.find({x, z});
+    if (found == regions.end()) {
+        return nullptr;
+    }
+    return found->second.get();
+}
+
+fs::path RegionsLayer::getRegionFilePath(int x, int z) const {
+    return folder / get_region_filename(x, z);
+}
+
+WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) {
+    if (auto region = getRegion(x, z)) {
+        return region;
+    }
+    std::lock_guard lock(mapMutex);
+    auto region_ptr = std::make_unique<WorldRegion>();
+    auto region = region_ptr.get();
+    regions[{x, z}] = std::move(region_ptr);
+    return region;
+}
+
+ubyte* RegionsLayer::getData(int x, int z, uint32_t& size, uint32_t& srcSize) {
+    int regionX, regionZ, localX, localZ;
+    calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
+
+    WorldRegion* region = getOrCreateRegion(regionX, regionZ);
+    ubyte* data = region->getChunkData(localX, localZ);
+    if (data == nullptr) {
+        auto regfile = getRegFile({regionX, regionZ});
+        if (regfile != nullptr) {
+            auto dataptr = readChunkData(x, z, size, srcSize, regfile.get());
+            if (dataptr) {
+                data = dataptr.get();
+                region->put(localX, localZ, std::move(dataptr), size, srcSize);
+            }
+        }
+    }
+    if (data != nullptr) {
+        auto sizevec = region->getChunkDataSize(localX, localZ);
+        size = sizevec[0];
+        srcSize = sizevec[1];
+        return data;
+    }
+    return nullptr;
+}
+
+void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) {
+    fs::path filename = folder / get_region_filename(x, z);
+
+    glm::ivec2 regcoord(x, z);
+    if (auto regfile = getRegFile(regcoord, false)) {
+        fetch_chunks(entry, x, z, regfile.get());
+
+        std::lock_guard lock(regFilesMutex);
+        regfile.reset();
+        closeRegFile(regcoord);
+    }
+
+    char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
+    header[8] = REGION_FORMAT_VERSION;
+    header[9] = static_cast<ubyte>(compression); // FIXME
+    std::ofstream file(filename, std::ios::out | std::ios::binary);
+    file.write(header, REGION_HEADER_SIZE);
+
+    size_t offset = REGION_HEADER_SIZE;
+    uint32_t intbuf;
+    uint offsets[REGION_CHUNKS_COUNT] {};
+
+    auto region = entry->getChunks();
+    auto sizes = entry->getSizes();
+
+    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
+        ubyte* chunk = region[i].get();
+        if (chunk == nullptr) {
+            continue;
+        }
+        offsets[i] = offset;
+
+        auto sizevec = sizes[i];
+        uint32_t compressedSize = sizevec[0];
+        uint32_t srcSize = sizevec[1];
+        
+        intbuf = dataio::h2le(compressedSize);
+        file.write(reinterpret_cast<const char*>(&intbuf), 4);
+        offset += 4;
+
+        intbuf = dataio::h2le(srcSize);
+        file.write(reinterpret_cast<const char*>(&intbuf), 4);
+        offset += 4;
+
+        file.write(reinterpret_cast<const char*>(chunk), compressedSize);
+        offset += compressedSize;
+    }
+    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
+        intbuf = dataio::h2le(offsets[i]);
+        file.write(reinterpret_cast<const char*>(&intbuf), 4);
+    }
+}
+
+std::unique_ptr<ubyte[]> RegionsLayer::readChunkData(
+    int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile
+) {
+    int regionX, regionZ, localX, localZ;
+    calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
+    int chunkIndex = localZ * REGION_SIZE + localX;
+    return rfile->read(chunkIndex, size, srcSize);
+}

src/files/WorldConverter.cpp

@@ -5,49 +5,140 @@
 #include <stdexcept>
 #include <utility>
 
-#include "content/ContentLUT.hpp"
+#include "content/ContentReport.hpp"
+#include "files/compatibility.hpp"
 #include "debug/Logger.hpp"
 #include "files/files.hpp"
 #include "objects/Player.hpp"
 #include "util/ThreadPool.hpp"
 #include "voxels/Chunk.hpp"
+#include "items/Inventory.hpp"
+#include "voxels/Block.hpp"
 #include "WorldFiles.hpp"
 
 namespace fs = std::filesystem;
 
 static debug::Logger logger("world-converter");
 
-class ConverterWorker : public util::Worker<convert_task, int> {
+class ConverterWorker : public util::Worker<ConvertTask, int> {
     std::shared_ptr<WorldConverter> converter;
 public:
     ConverterWorker(std::shared_ptr<WorldConverter> converter)
         : converter(std::move(converter)) {
     }
 
-    int operator()(const std::shared_ptr<convert_task>& task) override {
+    int operator()(const std::shared_ptr<ConvertTask>& task) override {
         converter->convert(*task);
         return 0;
     }
 };
 
+void WorldConverter::addRegionsTasks(
+    RegionLayerIndex layerid,
+    ConvertTaskType taskType
+) {
+    const auto& regions = wfile->getRegions();
+    auto regionsFolder = regions.getRegionsFolder(layerid);
+    if (!fs::is_directory(regionsFolder)) {
+        return;
+    }
+    for (const auto& file : fs::directory_iterator(regionsFolder)) {
+        int x, z;
+        std::string name = file.path().stem().string();
+        if (!WorldRegions::parseRegionFilename(name, x, z)) {
+            logger.error() << "could not parse region name " << name;
+            continue;
+        }
+        tasks.push(ConvertTask {taskType, file.path(), x, z, layerid});
+    }
+}
+
+void WorldConverter::createUpgradeTasks() {
+    const auto& regions = wfile->getRegions();
+    for (auto& issue : report->getIssues()) {
+        if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) {
+            continue;
+        }
+        addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION);
+    }
+}
+
+void WorldConverter::createConvertTasks() {
+    auto handleReorder = [=](ContentType contentType) {
+        switch (contentType) {
+            case ContentType::BLOCK:
+                addRegionsTasks(
+                    REGION_LAYER_VOXELS,
+                    ConvertTaskType::VOXELS
+                );
+                break;
+            case ContentType::ITEM:
+                addRegionsTasks(
+                    REGION_LAYER_INVENTORIES,
+                    ConvertTaskType::INVENTORIES
+                );
+                break;
+            default:
+                break;
+        }
+    };
+
+    const auto& regions = wfile->getRegions();
+    for (auto& issue : report->getIssues()) {
+        switch (issue.issueType) {
+            case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
+            case ContentIssueType::REGION_FORMAT_UPDATE:
+                break;
+            case ContentIssueType::MISSING:
+                throw std::runtime_error("issue can't be resolved");
+            case ContentIssueType::REORDER:
+                handleReorder(issue.contentType);
+                break;
+        }
+    }
+
+    tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()});
+}
+
+void WorldConverter::createBlockFieldsConvertTasks() {
+    // blocks data conversion requires correct block indices
+    // so it must be done AFTER voxels conversion
+    const auto& regions = wfile->getRegions();
+    for (auto& issue : report->getIssues()) {
+        switch (issue.issueType) {
+            case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
+                addRegionsTasks(
+                    REGION_LAYER_BLOCKS_DATA,
+                    ConvertTaskType::CONVERT_BLOCKS_DATA
+                );
+                break;
+            default:
+                break;
+        }
+    }
+}
+
 WorldConverter::WorldConverter(
     const std::shared_ptr<WorldFiles>& worldFiles,
     const Content* content,
-    std::shared_ptr<ContentLUT> lut
+    std::shared_ptr<ContentReport> reportPtr,
+    ConvertMode mode
 )
     : wfile(worldFiles),
-      lut(std::move(lut)),
+      report(std::move(reportPtr)),
-      content(content) {
+      content(content),
-    fs::path regionsFolder =
+      mode(mode)
-        wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS);
+{
-    if (!fs::is_directory(regionsFolder)) {
+    switch (mode) {
-        logger.error() << "nothing to convert";
+        case ConvertMode::UPGRADE:
-        return;
+            createUpgradeTasks();
-    }
+            break;
-    tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()}
+        case ConvertMode::REINDEX:
-    );
+            createConvertTasks();
-    for (const auto& file : fs::directory_iterator(regionsFolder)) {
+            break;
-        tasks.push(convert_task {convert_task_type::region, file.path()});
+        case ConvertMode::BLOCK_FIELDS:
+            createBlockFieldsConvertTasks();
+            break;
     }
 }
 
@@ -57,11 +148,13 @@ WorldConverter::~WorldConverter() {
 std::shared_ptr<Task> WorldConverter::startTask(
     const std::shared_ptr<WorldFiles>& worldFiles,
     const Content* content,
-    const std::shared_ptr<ContentLUT>& lut,
+    const std::shared_ptr<ContentReport>& report,
     const runnable& onDone,
+    ConvertMode mode,
     bool multithreading
 ) {
-    auto converter = std::make_shared<WorldConverter>(worldFiles, content, lut);
+    auto converter = std::make_shared<WorldConverter>(
+        worldFiles, content, report, mode);
     if (!multithreading) {
         converter->setOnComplete([=]() {
             converter->write();
@@ -69,15 +162,15 @@ std::shared_ptr<Task> WorldConverter::startTask(
         });
         return converter;
     }
-    auto pool = std::make_shared<util::ThreadPool<convert_task, int>>(
+    auto pool = std::make_shared<util::ThreadPool<ConvertTask, int>>(
         "converter-pool",
         [=]() { return std::make_shared<ConverterWorker>(converter); },
         [=](int&) {}
     );
     auto& converterTasks = converter->tasks;
     while (!converterTasks.empty()) {
-        const convert_task& task = converterTasks.front();
+        const ConvertTask& task = converterTasks.front();
-        auto ptr = std::make_shared<convert_task>(task);
+        auto ptr = std::make_shared<ConvertTask>(task);
         pool->enqueueJob(ptr);
         converterTasks.pop();
     }
@@ -88,39 +181,85 @@ std::shared_ptr<Task> WorldConverter::startTask(
     return pool;
 }
 
-void WorldConverter::convertRegion(const fs::path& file) const {
+void WorldConverter::upgradeRegion(
-    int x, z;
+    const fs::path& file, int x, int z, RegionLayerIndex layer
-    std::string name = file.stem().string();
+) const {
-    if (!WorldRegions::parseRegionFilename(name, x, z)) {
+    auto path = wfile->getRegions().getRegionFilePath(layer, x, z);
-        logger.error() << "could not parse name " << name;
+    auto bytes = files::read_bytes_buffer(path);
-        return;
+    auto buffer = compatibility::convert_region_2to3(bytes, layer);
-    }
+    files::write_bytes(path, buffer.data(), buffer.size());
-    logger.info() << "converting region " << name;
+}
-    wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) {
+
-        if (lut) {
+void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const {
-            Chunk::convert(data, lut.get());
+    logger.info() << "converting voxels region " << x << "_" << z;
-        }
+    wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS,
-        return true;
+    [=](std::unique_ptr<ubyte[]> data, uint32_t*) {
+        Chunk::convert(data.get(), report.get());
+        return data;
+    });
+}
+
+void WorldConverter::convertInventories(const fs::path& file, int x, int z) const {
+    logger.info() << "converting inventories region " << x << "_" << z;
+    wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) {
+        inventory->convert(report.get());
     });
 }
 
 void WorldConverter::convertPlayer(const fs::path& file) const {
     logger.info() << "converting player " << file.u8string();
     auto map = files::read_json(file);
-    Player::convert(map, lut.get());
+    Player::convert(map, report.get());
     files::write_json(file, map);
 }
 
-void WorldConverter::convert(const convert_task& task) const {
+void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const {
+    logger.info() << "converting blocks data";
+    wfile->getRegions().processBlocksData(x, z, 
+    [=](BlocksMetadata* heap, std::unique_ptr<ubyte[]> voxelsData) {
+        Chunk chunk(0, 0);
+        chunk.decode(voxelsData.get());
+
+        const auto& indices = content->getIndices()->blocks;
+
+        BlocksMetadata newHeap;
+        for (const auto& entry : *heap) {
+            size_t index = entry.index;
+            const auto& def = indices.require(chunk.voxels[index].id);
+            const auto& newStruct = *def.dataStruct;
+            const auto& found = report.blocksDataLayouts.find(def.name);
+            if (found == report.blocksDataLayouts.end()) {
+                logger.error() << "no previous fields layout found for block" 
+                    << def.name << " - discard";
+                continue; 
+            }
+            const auto& prevStruct = found->second;
+            uint8_t* dst = newHeap.allocate(index, newStruct.size());
+            newStruct.convert(prevStruct, entry.data(), dst, true);
+        }
+        *heap = std::move(newHeap);
+    });
+}
+
+void WorldConverter::convert(const ConvertTask& task) const {
     if (!fs::is_regular_file(task.file)) return;
 
     switch (task.type) {
-        case convert_task_type::region:
+        case ConvertTaskType::UPGRADE_REGION:
-            convertRegion(task.file);
+            upgradeRegion(task.file, task.x, task.z, task.layer);
             break;
-        case convert_task_type::player:
+        case ConvertTaskType::VOXELS:
+            convertVoxels(task.file, task.x, task.z);
+            break;
+        case ConvertTaskType::INVENTORIES:
+            convertInventories(task.file, task.x, task.z);
+            break;
+        case ConvertTaskType::PLAYER:
             convertPlayer(task.file);
             break;
+        case ConvertTaskType::CONVERT_BLOCKS_DATA:
+            convertBlocksData(task.x, task.z, *report);
+            break;
     }
 }
 
@@ -128,7 +267,7 @@ void WorldConverter::convertNext() {
     if (tasks.empty()) {
         throw std::runtime_error("no more regions to convert");
     }
-    convert_task task = tasks.front();
+    ConvertTask task = tasks.front();
     tasks.pop();
     tasksDone++;
 
@@ -155,8 +294,22 @@ bool WorldConverter::isActive() const {
 }
 
 void WorldConverter::write() {
-    logger.info() << "writing world";
+    logger.info() << "applying changes";
-    wfile->write(nullptr, content);
+
+    auto patch = dv::object();
+    switch (mode) {
+        case ConvertMode::UPGRADE:
+            patch["region-version"] = REGION_FORMAT_VERSION;
+            break;
+        case ConvertMode::REINDEX:
+            WorldFiles::createContentIndicesCache(content->getIndices(), patch);
+            break;
+        case ConvertMode::BLOCK_FIELDS:
+            WorldFiles::createBlockFieldsIndices(content->getIndices(), patch);
+            break;
+    }
+    wfile->patchIndicesFile(patch);
+    wfile->write(nullptr, nullptr);
 }
 
 void WorldConverter::waitForEnd() {

src/files/WorldConverter.hpp

@@ -6,40 +6,77 @@
 
 #include "delegates.hpp"
 #include "interfaces/Task.hpp"
+#include "files/world_regions_fwd.hpp"
 #include "typedefs.hpp"
 
 namespace fs = std::filesystem;
 
 class Content;
-class ContentLUT;
+class ContentReport;
 class WorldFiles;
 
-enum class convert_task_type { region, player };
+enum class ConvertTaskType {
+    /// @brief rewrite voxels region indices
+    VOXELS,
+    /// @brief rewrite inventories region indices
+    INVENTORIES,
+    /// @brief rewrite player
+    PLAYER,
+    /// @brief refresh region file version
+    UPGRADE_REGION,
+    /// @brief convert blocks data to updated layouts
+    CONVERT_BLOCKS_DATA,
+};
 
-struct convert_task {
+struct ConvertTask {
-    convert_task_type type;
+    ConvertTaskType type;
     fs::path file;
+
+    /// @brief region coords
+    int x, z;
+    RegionLayerIndex layer;
+};
+
+enum class ConvertMode {
+    UPGRADE,
+    REINDEX,
+    BLOCK_FIELDS,
 };
 
 class WorldConverter : public Task {
     std::shared_ptr<WorldFiles> wfile;
-    std::shared_ptr<ContentLUT> const lut;
+    std::shared_ptr<ContentReport> const report;
     const Content* const content;
-    std::queue<convert_task> tasks;
+    std::queue<ConvertTask> tasks;
     runnable onComplete;
     uint tasksDone = 0;
+    ConvertMode mode;
 
+    void upgradeRegion(
+        const fs::path& file, int x, int z, RegionLayerIndex layer) const;
     void convertPlayer(const fs::path& file) const;
-    void convertRegion(const fs::path& file) const;
+    void convertVoxels(const fs::path& file, int x, int z) const;
+    void convertInventories(const fs::path& file, int x, int z) const;
+    void convertBlocksData(int x, int z, const ContentReport& report) const;
+
+    void addRegionsTasks(
+        RegionLayerIndex layerid,
+        ConvertTaskType taskType
+    );
+
+    void createUpgradeTasks();
+    void createConvertTasks();
+    void createBlockFieldsConvertTasks();
 public:
     WorldConverter(
         const std::shared_ptr<WorldFiles>& worldFiles,
         const Content* content,
-        std::shared_ptr<ContentLUT> lut
+        std::shared_ptr<ContentReport> report,
+        ConvertMode mode
     );
     ~WorldConverter();
 
-    void convert(const convert_task& task) const;
+    void convert(const ConvertTask& task) const;
     void convertNext();
     void setOnComplete(runnable callback);
     void write();
@@ -54,8 +91,9 @@ public:
     static std::shared_ptr<Task> startTask(
         const std::shared_ptr<WorldFiles>& worldFiles,
         const Content* content,
-        const std::shared_ptr<ContentLUT>& lut,
+        const std::shared_ptr<ContentReport>& report,
         const runnable& onDone,
+        ConvertMode mode,
         bool multithreading
     );
 };

src/files/WorldFiles.cpp

@@ -21,9 +21,11 @@
 #include "objects/EntityDef.hpp"
 #include "objects/Player.hpp"
 #include "physics/Hitbox.hpp"
+#include "data/StructLayout.hpp"
 #include "settings.hpp"
 #include "typedefs.hpp"
 #include "util/data_io.hpp"
+#include "util/stringutil.hpp"
 #include "voxels/Block.hpp"
 #include "voxels/Chunk.hpp"
 #include "voxels/voxel.hpp"
@@ -73,7 +75,9 @@ fs::path WorldFiles::getPacksFile() const {
     return directory / fs::path("packs.list");
 }
 
-void WorldFiles::write(const World* world, const Content* content) {
+void WorldFiles::write(
+    const World* world, const Content* content
+) {
     if (world) {
         writeWorldInfo(world->getInfo());
         if (!fs::exists(getPacksFile())) {
@@ -83,9 +87,10 @@ void WorldFiles::write(const World* world, const Content* content) {
     if (generatorTestMode) {
         return;
     }
-
+    if (content) {
-    writeIndices(content->getIndices());
+        writeIndices(content->getIndices());
-    regions.write();
+    }
+    regions.writeAll();
 }
 
 void WorldFiles::writePacks(const std::vector<ContentPack>& packs) {
@@ -107,11 +112,33 @@ static void write_indices(
     }
 }
 
-void WorldFiles::writeIndices(const ContentIndices* indices) {
+void WorldFiles::createContentIndicesCache(
-    dv::value root = dv::object();
+    const ContentIndices* indices, dv::value& root
+) {
     write_indices(indices->blocks, root.list("blocks"));
     write_indices(indices->items, root.list("items"));
     write_indices(indices->entities, root.list("entities"));
+}
+
+void WorldFiles::createBlockFieldsIndices(
+    const ContentIndices* indices, dv::value& root
+) {
+    auto& structsMap = root.object("blocks-data");
+    for (const auto* def : indices->blocks.getIterable()) {
+        if (def->dataStruct == nullptr) {
+            continue;
+        }
+        structsMap[def->name] = def->dataStruct->serialize();
+    }
+}
+
+void WorldFiles::writeIndices(const ContentIndices* indices) {
+    dv::value root = dv::object();
+    root["region-version"] = REGION_FORMAT_VERSION;
+
+    createContentIndicesCache(indices, root);
+    createBlockFieldsIndices(indices, root);
+    
     files::write_json(getIndicesFile(), root);
 }
 
@@ -164,6 +191,20 @@ bool WorldFiles::readResourcesData(const Content* content) {
     return true;
 }
 
+void WorldFiles::patchIndicesFile(const dv::value& map) {
+    fs::path file = getIndicesFile();
+    if (!fs::is_regular_file(file)) {
+        logger.error() << file.filename().u8string() << " does not exists";
+        return;
+    }
+    auto root = files::read_json(file);
+    for (const auto& [key, value] : map.asObject()) {
+        logger.info() << "patching indices.json: update " << util::quote(key);
+        root[key] = value;
+    }
+    files::write_json(file, root, true);
+}
+
 static void erase_pack_indices(dv::value& root, const std::string& id) {
     auto prefix = id + ":";
     auto& blocks = root["blocks"];

src/files/WorldFiles.hpp

@@ -51,6 +51,15 @@ public:
     std::optional<WorldInfo> readWorldInfo();
     bool readResourcesData(const Content* content);
 
+    static void createContentIndicesCache(
+        const ContentIndices* indices, dv::value& root
+    );
+    static void createBlockFieldsIndices(
+        const ContentIndices* indices, dv::value& root
+    );
+
+    void patchIndicesFile(const dv::value& map);
+
     /// @brief Write all unsaved data to world files
     /// @param world target world
     /// @param content world content
@@ -63,8 +72,6 @@ public:
     /// @return world folder
     fs::path getFolder() const;
 
-    static const inline std::string WORLD_FILE = "world.json";
-
     WorldRegions& getRegions() {
         return regions;
     }
@@ -72,4 +79,6 @@ public:
     bool doesWriteLights() const {
         return doWriteLights;
     }
+
+    static const inline std::string WORLD_FILE = "world.json";
 };

src/files/WorldRegions.cpp

@@ -4,60 +4,24 @@
 #include <utility>
 #include <vector>
 
+#include "debug/Logger.hpp"
 #include "coders/json.hpp"
 #include "coders/byte_utils.hpp"
 #include "coders/rle.hpp"
+#include "coders/binary_json.hpp"
 #include "items/Inventory.hpp"
 #include "maths/voxmaths.hpp"
 #include "util/data_io.hpp"
-#include "coders/binary_json.hpp"
 
 #define REGION_FORMAT_MAGIC ".VOXREG"
 
-regfile::regfile(fs::path filename) : file(std::move(filename)) {
+static debug::Logger logger("world-regions");
-    if (file.length() < REGION_HEADER_SIZE)
-        throw std::runtime_error("incomplete region file header");
-    char header[REGION_HEADER_SIZE];
-    file.read(header, REGION_HEADER_SIZE);
-
-    // avoid of use strcmp_s
-    if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) !=
-        REGION_FORMAT_MAGIC) {
-        throw std::runtime_error("invalid region file magic number");
-    }
-    version = header[8];
-    if (static_cast<uint>(version) > REGION_FORMAT_VERSION) {
-        throw illegal_region_format(
-            "region format " + std::to_string(version) + " is not supported"
-        );
-    }
-}
-
-std::unique_ptr<ubyte[]> regfile::read(int index, uint32_t& length) {
-    size_t file_size = file.length();
-    size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
-
-    uint32_t offset;
-    file.seekg(table_offset + index * 4);
-    file.read(reinterpret_cast<char*>(&offset), 4);
-    offset = dataio::read_int32_big(reinterpret_cast<const ubyte*>(&offset), 0);
-    if (offset == 0) {
-        return nullptr;
-    }
-
-    file.seekg(offset);
-    file.read(reinterpret_cast<char*>(&offset), 4);
-    length = dataio::read_int32_big(reinterpret_cast<const ubyte*>(&offset), 0);
-    auto data = std::make_unique<ubyte[]>(length);
-    file.read(reinterpret_cast<char*>(data.get()), length);
-    return data;
-}
 
 WorldRegion::WorldRegion()
     : chunksData(
           std::make_unique<std::unique_ptr<ubyte[]>[]>(REGION_CHUNKS_COUNT)
       ),
-      sizes(std::make_unique<uint32_t[]>(REGION_CHUNKS_COUNT)) {
+      sizes(std::make_unique<glm::u32vec2[]>(REGION_CHUNKS_COUNT)) {
 }
 
 WorldRegion::~WorldRegion() = default;
@@ -73,293 +37,89 @@ std::unique_ptr<ubyte[]>* WorldRegion::getChunks() const {
     return chunksData.get();
 }
 
-uint32_t* WorldRegion::getSizes() const {
+glm::u32vec2* WorldRegion::getSizes() const {
     return sizes.get();
 }
 
-void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) {
+void WorldRegion::put(
+    uint x, uint z, std::unique_ptr<ubyte[]> data, uint32_t size, uint32_t srcSize
+) {
     size_t chunk_index = z * REGION_SIZE + x;
-    chunksData[chunk_index].reset(data);
+    chunksData[chunk_index] = std::move(data);
-    sizes[chunk_index] = size;
+    sizes[chunk_index] = glm::u32vec2(size, srcSize);
 }
 
 ubyte* WorldRegion::getChunkData(uint x, uint z) {
     return chunksData[z * REGION_SIZE + x].get();
 }
 
-uint WorldRegion::getChunkDataSize(uint x, uint z) {
+glm::u32vec2 WorldRegion::getChunkDataSize(uint x, uint z) {
     return sizes[z * REGION_SIZE + x];
 }
 
 WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
-    for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) {
+    for (size_t i = 0; i < REGION_LAYERS_COUNT; i++) {
-        layers[i].layer = i;
+        layers[i].layer = static_cast<RegionLayerIndex>(i);
     }
-    layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions");
+    auto& voxels = layers[REGION_LAYER_VOXELS];
-    layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights");
+    voxels.folder = directory / fs::path("regions");
+    voxels.compression = compression::Method::EXTRLE16;
+
+    auto& lights = layers[REGION_LAYER_LIGHTS];
+    lights.folder = directory / fs::path("lights");
+    lights.compression = compression::Method::EXTRLE8;
+
     layers[REGION_LAYER_INVENTORIES].folder =
         directory / fs::path("inventories");
     layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities");
+
+    auto& blocksData = layers[REGION_LAYER_BLOCKS_DATA];
+    blocksData.folder = directory / fs::path("blocksdata");
 }
 
 WorldRegions::~WorldRegions() = default;
 
-WorldRegion* WorldRegions::getRegion(int x, int z, int layer) {
+void RegionsLayer::writeAll() {
-    RegionsLayer& regions = layers[layer];
+    for (auto& it : regions) {
-    std::lock_guard lock(regions.mutex);
-    auto found = regions.regions.find(glm::ivec2(x, z));
-    if (found == regions.regions.end()) {
-        return nullptr;
-    }
-    return found->second.get();
-}
-
-WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) {
-    if (auto region = getRegion(x, z, layer)) {
-        return region;
-    }
-    RegionsLayer& regions = layers[layer];
-    std::lock_guard lock(regions.mutex);
-    auto region_ptr = std::make_unique<WorldRegion>();
-    auto region = region_ptr.get();
-    regions.regions[{x, z}] = std::move(region_ptr);
-    return region;
-}
-
-std::unique_ptr<ubyte[]> WorldRegions::compress(
-    const ubyte* src, size_t srclen, size_t& len
-) {
-    auto buffer = bufferPool.get();
-    auto bytes = buffer.get();
-
-    len = extrle::encode(src, srclen, bytes);
-    auto data = std::make_unique<ubyte[]>(len);
-    for (size_t i = 0; i < len; i++) {
-        data[i] = bytes[i];
-    }
-    return data;
-}
-
-std::unique_ptr<ubyte[]> WorldRegions::decompress(
-    const ubyte* src, size_t srclen, size_t dstlen
-) {
-    auto decompressed = std::make_unique<ubyte[]>(dstlen);
-    extrle::decode(src, srclen, decompressed.get());
-    return decompressed;
-}
-
-inline void calc_reg_coords(
-    int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
-) {
-    regionX = floordiv(x, REGION_SIZE);
-    regionZ = floordiv(z, REGION_SIZE);
-    localX = x - (regionX * REGION_SIZE);
-    localZ = z - (regionZ * REGION_SIZE);
-}
-
-std::unique_ptr<ubyte[]> WorldRegions::readChunkData(
-    int x, int z, uint32_t& length, regfile* rfile
-) {
-    int regionX, regionZ, localX, localZ;
-    calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
-    int chunkIndex = localZ * REGION_SIZE + localX;
-    return rfile->read(chunkIndex, length);
-}
-
-/// @brief Read missing chunks data (null pointers) from region file
-void WorldRegions::fetchChunks(
-    WorldRegion* region, int x, int z, regfile* file
-) {
-    auto* chunks = region->getChunks();
-    uint32_t* sizes = region->getSizes();
-
-    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
-        int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
-        int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
-        if (chunks[i] == nullptr) {
-            chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file);
-        }
-    }
-}
-
-ubyte* WorldRegions::getData(int x, int z, int layer, uint32_t& size) {
-    if (generatorTestMode) {
-        return nullptr;
-    }
-    int regionX, regionZ, localX, localZ;
-    calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
-
-    WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
-    ubyte* data = region->getChunkData(localX, localZ);
-    if (data == nullptr) {
-        auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer));
-        if (regfile != nullptr) {
-            data = readChunkData(x, z, size, regfile.get()).release();
-        }
-        if (data != nullptr) {
-            region->put(localX, localZ, data, size);
-        }
-    }
-    if (data != nullptr) {
-        size = region->getChunkDataSize(localX, localZ);
-        return data;
-    }
-    return nullptr;
-}
-
-regfile_ptr WorldRegions::useRegFile(glm::ivec3 coord) {
-    auto* file = openRegFiles[coord].get();
-    file->inUse = true;
-    return regfile_ptr(file, &regFilesCv);
-}
-
-void WorldRegions::closeRegFile(glm::ivec3 coord) {
-    openRegFiles.erase(coord);
-    regFilesCv.notify_one();
-}
-
-// Marks regfile as used and unmarks when shared_ptr dies
-regfile_ptr WorldRegions::getRegFile(glm::ivec3 coord, bool create) {
-    {
-        std::lock_guard lock(regFilesMutex);
-        const auto found = openRegFiles.find(coord);
-        if (found != openRegFiles.end()) {
-            if (found->second->inUse) {
-                throw std::runtime_error("regfile is currently in use");
-            }
-            return useRegFile(found->first);
-        }
-    }
-    if (create) {
-        return createRegFile(coord);
-    }
-    return nullptr;
-}
-
-regfile_ptr WorldRegions::createRegFile(glm::ivec3 coord) {
-    fs::path file =
-        layers[coord[2]].folder / getRegionFilename(coord[0], coord[1]);
-    if (!fs::exists(file)) {
-        return nullptr;
-    }
-    if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
-        std::unique_lock lock(regFilesMutex);
-        while (true) {
-            bool closed = false;
-            // FIXME: bad choosing algorithm
-            for (auto& entry : openRegFiles) {
-                if (!entry.second->inUse) {
-                    closeRegFile(entry.first);
-                    closed = true;
-                    break;
-                }
-            }
-            if (closed) {
-                break;
-            }
-            // notified when any regfile gets out of use or closed
-            regFilesCv.wait(lock);
-        }
-        openRegFiles[coord] = std::make_unique<regfile>(file);
-        return useRegFile(coord);
-    } else {
-        std::lock_guard lock(regFilesMutex);
-        openRegFiles[coord] = std::make_unique<regfile>(file);
-        return useRegFile(coord);
-    }
-}
-
-fs::path WorldRegions::getRegionFilename(int x, int z) const {
-    return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
-}
-
-void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry) {
-    fs::path filename = layers[layer].folder / getRegionFilename(x, z);
-
-    glm::ivec3 regcoord(x, z, layer);
-    if (auto regfile = getRegFile(regcoord, false)) {
-        fetchChunks(entry, x, z, regfile.get());
-
-        std::lock_guard lock(regFilesMutex);
-        regfile.reset();
-        closeRegFile(regcoord);
-    }
-
-    char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
-    header[8] = REGION_FORMAT_VERSION;
-    header[9] = 0;  // flags
-    std::ofstream file(filename, std::ios::out | std::ios::binary);
-    file.write(header, REGION_HEADER_SIZE);
-
-    size_t offset = REGION_HEADER_SIZE;
-    char intbuf[4] {};
-    uint offsets[REGION_CHUNKS_COUNT] {};
-
-    auto* region = entry->getChunks();
-    uint32_t* sizes = entry->getSizes();
-
-    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
-        ubyte* chunk = region[i].get();
-        if (chunk == nullptr) {
-            offsets[i] = 0;
-        } else {
-            offsets[i] = offset;
-
-            size_t compressedSize = sizes[i];
-            dataio::write_int32_big(
-                compressedSize, reinterpret_cast<ubyte*>(intbuf), 0
-            );
-            offset += 4 + compressedSize;
-
-            file.write(intbuf, 4);
-            file.write(reinterpret_cast<const char*>(chunk), compressedSize);
-        }
-    }
-    for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
-        dataio::write_int32_big(
-            offsets[i], reinterpret_cast<ubyte*>(intbuf), 0
-        );
-        file.write(intbuf, 4);
-    }
-}
-
-void WorldRegions::writeRegions(int layer) {
-    for (auto& it : layers[layer].regions) {
         WorldRegion* region = it.second.get();
         if (region->getChunks() == nullptr || !region->isUnsaved()) {
             continue;
         }
-        glm::ivec2 key = it.first;
+        const auto& key = it.first;
-        writeRegion(key[0], key[1], layer, region);
+        writeRegion(key[0], key[1], region);
     }
 }
 
 void WorldRegions::put(
     int x,
     int z,
-    int layer,
+    RegionLayerIndex layerid,
     std::unique_ptr<ubyte[]> data,
-    size_t size,
+    size_t srcSize
-    bool rle
 ) {
-    if (rle) {
+    size_t size = srcSize;
-        size_t compressedSize;
+    auto& layer = layers[layerid];
-        auto compressed = compress(data.get(), size, compressedSize);
-        put(x, z, layer, std::move(compressed), compressedSize, false);
-        return;
-    }
     int regionX, regionZ, localX, localZ;
     calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
 
-    WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
+    WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
     region->setUnsaved(true);
-    region->put(localX, localZ, data.release(), size);
+    
+    if (data == nullptr) {
+        region->put(localX, localZ, nullptr, 0, 0);
+        return;
+    }
+
+    if (layer.compression != compression::Method::NONE) {
+        data = compression::compress(
+            data.get(), size, size, layer.compression);
+    }
+    region->put(localX, localZ, std::move(data), size, srcSize);
 }
 
 static std::unique_ptr<ubyte[]> write_inventories(
-    Chunk* chunk, uint& datasize
+    const ChunkInventoriesMap& inventories, uint32_t& datasize
 ) {
-    auto& inventories = chunk->inventories;
     ByteBuilder builder;
     builder.putInt32(inventories.size());
     for (auto& entry : inventories) {
@@ -376,7 +136,22 @@ static std::unique_ptr<ubyte[]> write_inventories(
     return data;
 }
 
-/// @brief Store chunk data (voxels and lights) in region (existing or new)
+static ChunkInventoriesMap load_inventories(const ubyte* src, uint32_t size) {
+    ChunkInventoriesMap inventories;
+    ByteReader reader(src, size);
+    auto count = reader.getInt32();
+    for (int i = 0; i < count; i++) {
+        uint index = reader.getInt32();
+        uint size = reader.getInt32();
+        auto map = json::from_binary(reader.pointer(), size);
+        reader.skip(size);
+        auto inv = std::make_shared<Inventory>(0, 0);
+        inv->deserialize(map);
+        inventories[index] = inv;
+    }
+    return inventories;
+}
+
 void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
     assert(chunk != nullptr);
     if (!chunk->flags.lighted) {
@@ -394,8 +169,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
         chunk->z,
         REGION_LAYER_VOXELS,
         chunk->encode(),
-        CHUNK_DATA_LEN,
+        CHUNK_DATA_LEN);
-        true);
 
     // Writing lights cache
     if (doWriteLights && chunk->flags.lighted) {
@@ -403,19 +177,17 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
             chunk->z,
             REGION_LAYER_LIGHTS,
             chunk->lightmap.encode(),
-            LIGHTMAP_DATA_LEN,
+            LIGHTMAP_DATA_LEN);
-            true);
     }
     // Writing block inventories
     if (!chunk->inventories.empty()) {
         uint datasize;
-        auto data = write_inventories(chunk, datasize);
+        auto data = write_inventories(chunk->inventories, datasize);
         put(chunk->x,
             chunk->z,
             REGION_LAYER_INVENTORIES,
             std::move(data),
-            datasize,
+            datasize);
-            false);
     }
     // Writing entities
     if (!entitiesData.empty()) {
@@ -427,71 +199,164 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
             chunk->z,
             REGION_LAYER_ENTITIES,
             std::move(data),
-            entitiesData.size(),
+            entitiesData.size());
-            false);
+    }
+    // Writing blocks data
+    if (chunk->flags.blocksData) {
+        auto bytes = chunk->blocksMetadata.serialize();
+        put(chunk->x,
+            chunk->z,
+            REGION_LAYER_BLOCKS_DATA,
+            bytes.release(),
+            bytes.size());
     }
 }
 
-std::unique_ptr<ubyte[]> WorldRegions::getChunk(int x, int z) {
+std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
     uint32_t size;
-    auto* data = getData(x, z, REGION_LAYER_VOXELS, size);
+    uint32_t srcSize;
+    auto& layer = layers[REGION_LAYER_VOXELS];
+    auto* data = layer.getData(x, z, size, srcSize);
     if (data == nullptr) {
         return nullptr;
     }
-    return decompress(data, size, CHUNK_DATA_LEN);
+    assert(srcSize == CHUNK_DATA_LEN);
+    return compression::decompress(data, size, srcSize, layer.compression);
 }
 
-/// @brief Get cached lights for chunk at x,z
-/// @return lights data or nullptr
 std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
     uint32_t size;
-    auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size);
+    uint32_t srcSize;
+    auto& layer = layers[REGION_LAYER_LIGHTS];
+    auto* bytes = layer.getData(x, z, size, srcSize);
     if (bytes == nullptr) {
         return nullptr;
     }
-    auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
+    auto data = compression::decompress(
+        bytes, size, srcSize, layer.compression
+    );
+    assert(srcSize == LIGHTMAP_DATA_LEN);
     return Lightmap::decode(data.get());
 }
 
-chunk_inventories_map WorldRegions::fetchInventories(int x, int z) {
+ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) {
-    chunk_inventories_map meta;
     uint32_t bytesSize;
-    const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize);
+    uint32_t srcSize;
-    if (data == nullptr) {
+    auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize, srcSize);
-        return meta;
+    if (bytes == nullptr) {
+        return {};
     }
-    ByteReader reader(data, bytesSize);
+    return load_inventories(bytes, bytesSize);
-    auto count = reader.getInt32();
+}
-    for (int i = 0; i < count; i++) {
+
-        uint index = reader.getInt32();
+BlocksMetadata WorldRegions::getBlocksData(int x, int z) {
-        uint size = reader.getInt32();
+    uint32_t bytesSize;
-        auto map = json::from_binary(reader.pointer(), size);
+    uint32_t srcSize;
-        reader.skip(size);
+    auto bytes = layers[REGION_LAYER_BLOCKS_DATA].getData(x, z, bytesSize, srcSize);
-        auto inv = std::make_shared<Inventory>(0, 0);
+    if (bytes == nullptr) {
-        inv->deserialize(map);
+        return {};
-        meta[index] = inv;
+    }
+    BlocksMetadata heap;
+    heap.deserialize(bytes, bytesSize);
+    return heap;
+}
+
+void WorldRegions::processInventories(int x, int z, const InventoryProc& func) {
+    processRegion(x, z, REGION_LAYER_INVENTORIES,
+    [=](std::unique_ptr<ubyte[]> data, uint32_t* size) {
+        auto inventories = load_inventories(data.get(), *size);
+        for (const auto& [_, inventory] : inventories) {
+            func(inventory.get());
+        }
+        return write_inventories(inventories, *size);
+    });
+}
+
+void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) {
+    auto& voxLayer = layers[REGION_LAYER_VOXELS];
+    auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA];
+    if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) {
+        throw std::runtime_error("not implemented for in-memory regions");
+    }
+    auto datRegfile = datLayer.getRegFile({x, z});
+    if (datRegfile == nullptr) {
+        throw std::runtime_error("could not open region file");
+    }
+    auto voxRegfile = voxLayer.getRegFile({x, z});
+    if (voxRegfile == nullptr) {
+        logger.warning() << "missing voxels region - discard blocks data for "
+            << x << "_" << z;
+        deleteRegion(REGION_LAYER_BLOCKS_DATA, x, z);
+        return;
+    }
+    for (uint cz = 0; cz < REGION_SIZE; cz++) {
+        for (uint cx = 0; cx < REGION_SIZE; cx++) {
+            int gx = cx + x * REGION_SIZE;
+            int gz = cz + z * REGION_SIZE;
+
+            uint32_t datLength;
+            uint32_t datSrcSize;
+            auto datData = RegionsLayer::readChunkData(
+                gx, gz, datLength, datSrcSize, datRegfile.get()
+            );
+            if (datData == nullptr) {
+                continue;
+            }
+            uint32_t voxLength;
+            uint32_t voxSrcSize;
+            auto voxData = RegionsLayer::readChunkData(
+                gx, gz, voxLength, voxSrcSize, voxRegfile.get()
+            );
+            if (voxData == nullptr) {
+                logger.warning()
+                    << "missing voxels for chunk (" << gx << ", " << gz << ")";
+                put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0);
+                continue;
+            }
+            voxData = compression::decompress(
+                voxData.get(), voxLength, voxSrcSize, voxLayer.compression
+            );
+
+            BlocksMetadata blocksData;
+            blocksData.deserialize(datData.get(), datLength);
+            try {
+                func(&blocksData, std::move(voxData));
+            } catch (const std::exception& err) {
+                logger.error() << "an error ocurred while processing blocks "
+                    "data in chunk (" << gx << ", " << gz << "): " << err.what();
+                blocksData = {};
+            }
+            auto bytes = blocksData.serialize();
+            put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size());
+        }
     }
-    return meta;
 }
 
 dv::value WorldRegions::fetchEntities(int x, int z) {
+    if (generatorTestMode) {
+        return nullptr;
+    }
     uint32_t bytesSize;
-    const ubyte* data = getData(x, z, REGION_LAYER_ENTITIES, bytesSize);
+    uint32_t srcSize;
+    const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize, srcSize);
     if (data == nullptr) {
         return nullptr;
     }
     auto map = json::from_binary(data, bytesSize);
-    if (map.size() == 0) {
+    if (map.empty()) {
         return nullptr;
     }
     return map;
 }
 
-void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
+void WorldRegions::processRegion(
-    if (getRegion(x, z, REGION_LAYER_VOXELS)) {
+    int x, int z, RegionLayerIndex layerid, const RegionProc& func
+) {
+    auto& layer = layers[layerid];
+    if (layer.getRegion(x, z)) {
         throw std::runtime_error("not implemented for in-memory regions");
     }
-    auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS));
+    auto regfile = layer.getRegFile({x, z});
     if (regfile == nullptr) {
         throw std::runtime_error("could not open region file");
     }
@@ -500,31 +365,50 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
             int gx = cx + x * REGION_SIZE;
             int gz = cz + z * REGION_SIZE;
             uint32_t length;
-            auto data = readChunkData(gx, gz, length, regfile.get());
+            uint32_t srcSize;
+            auto data =
+                RegionsLayer::readChunkData(gx, gz, length, srcSize, regfile.get());
             if (data == nullptr) {
                 continue;
             }
-            data = decompress(data.get(), length, CHUNK_DATA_LEN);
+            if (layer.compression != compression::Method::NONE) {
-            if (func(data.get())) {
+                data = compression::decompress(
-                put(gx,
+                    data.get(), length, srcSize, layer.compression
-                    gz,
+                );
-                    REGION_LAYER_VOXELS,
+            } else {
-                    std::move(data),
+                srcSize = length;
-                    CHUNK_DATA_LEN,
+            }
-                    true);
+            if (auto writeData = func(std::move(data), &srcSize)) {
+                put(gx, gz, layerid, std::move(writeData), srcSize);
             }
         }
     }
 }
 
-fs::path WorldRegions::getRegionsFolder(int layer) const {
+const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const {
-    return layers[layer].folder;
+    return layers[layerid].folder;
 }
 
-void WorldRegions::write() {
+fs::path WorldRegions::getRegionFilePath(RegionLayerIndex layerid, int x, int z) const {
+    return layers[layerid].getRegionFilePath(x, z);
+}
+
+void WorldRegions::writeAll() {
     for (auto& layer : layers) {
         fs::create_directories(layer.folder);
-        writeRegions(layer.layer);
+        layer.writeAll();
+    }
+}
+
+void WorldRegions::deleteRegion(RegionLayerIndex layerid, int x, int z) {
+    auto& layer = layers[layerid];
+    if (layer.getRegFile({x, z}, false)) {
+        throw std::runtime_error("region file is currently in use");
+    }
+    auto file = layer.getRegionFilePath(x, z);
+    if (fs::exists(file)) {
+        logger.info() << "remove region file " << file.u8string();
+        fs::remove(file);
     }
 }
 

src/files/WorldRegions.hpp

@@ -11,7 +11,10 @@
 #include "typedefs.hpp"
 #include "util/BufferPool.hpp"
 #include "voxels/Chunk.hpp"
+#include "maths/voxmaths.hpp"
+#include "coders/compression.hpp"
 #include "files.hpp"
+#include "world_regions_fwd.hpp"
 
 #define GLM_ENABLE_EXPERIMENTAL
 #include <glm/gtx/hash.hpp>
@@ -20,16 +23,9 @@ namespace fs = std::filesystem;
 
 inline constexpr uint REGION_HEADER_SIZE = 10;
 
-inline constexpr uint REGION_LAYER_VOXELS = 0;
-inline constexpr uint REGION_LAYER_LIGHTS = 1;
-inline constexpr uint REGION_LAYER_INVENTORIES = 2;
-inline constexpr uint REGION_LAYER_ENTITIES = 3;
-
 inline constexpr uint REGION_SIZE_BIT = 5;
 inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT));
 inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE));
-inline constexpr uint REGION_FORMAT_VERSION = 2;
-inline constexpr uint MAX_OPEN_REGION_FILES = 16;
 
 class illegal_region_format : public std::runtime_error {
 public:
@@ -40,21 +36,21 @@ public:
 
 class WorldRegion {
     std::unique_ptr<std::unique_ptr<ubyte[]>[]> chunksData;
-    std::unique_ptr<uint32_t[]> sizes;
+    std::unique_ptr<glm::u32vec2[]> sizes;
     bool unsaved = false;
 public:
     WorldRegion();
     ~WorldRegion();
 
-    void put(uint x, uint z, ubyte* data, uint32_t size);
+    void put(uint x, uint z, std::unique_ptr<ubyte[]> data, uint32_t size, uint32_t srcSize);
     ubyte* getChunkData(uint x, uint z);
-    uint getChunkDataSize(uint x, uint z);
+    glm::u32vec2 getChunkDataSize(uint x, uint z);
 
     void setUnsaved(bool unsaved);
     bool isUnsaved() const;
 
     std::unique_ptr<ubyte[]>* getChunks() const;
-    uint32_t* getSizes() const;
+    glm::u32vec2* getSizes() const;
 };
 
 struct regfile {
@@ -65,19 +61,15 @@ struct regfile {
     regfile(fs::path filename);
     regfile(const regfile&) = delete;
 
-    std::unique_ptr<ubyte[]> read(int index, uint32_t& length);
+    std::unique_ptr<ubyte[]> read(int index, uint32_t& size, uint32_t& srcSize);
 };
 
-using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
+using RegionsMap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
-using regionproc = std::function<bool(ubyte*)>;
+using RegionProc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
-
+using InventoryProc = std::function<void(Inventory*)>;
-struct RegionsLayer {
+using BlockDataProc = std::function<void(BlocksMetadata*, std::unique_ptr<ubyte[]>)>;
-    int layer;
-    fs::path folder;
-    regionsmap regions;
-    std::mutex mutex;
-};
 
+/// @brief Region file pointer keeping inUse flag on until destroyed
 class regfile_ptr {
     regfile* file;
     std::condition_variable* cv;
@@ -115,58 +107,80 @@ public:
     }
 };
 
-class WorldRegions {
+inline void calc_reg_coords(
-    fs::path directory;
+    int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
-    std::unordered_map<glm::ivec3, std::unique_ptr<regfile>> openRegFiles;
+) {
+    regionX = floordiv(x, REGION_SIZE);
+    regionZ = floordiv(z, REGION_SIZE);
+    localX = x - (regionX * REGION_SIZE);
+    localZ = z - (regionZ * REGION_SIZE);
+}
+
+struct RegionsLayer {
+    /// @brief Layer index
+    RegionLayerIndex layer;
+    
+    /// @brief Regions layer folder
+    fs::path folder;
+
+    compression::Method compression = compression::Method::NONE;
+
+    /// @brief In-memory regions data
+    RegionsMap regions;
+
+    /// @brief In-memory regions map mutex
+    std::mutex mapMutex;
+
+    /// @brief Open region files map
+    std::unordered_map<glm::ivec2, std::unique_ptr<regfile>> openRegFiles;
+
+    /// @brief Open region files map mutex
     std::mutex regFilesMutex;
     std::condition_variable regFilesCv;
-    RegionsLayer layers[4] {};
-    util::BufferPool<ubyte> bufferPool {
-        std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2};
 
-    WorldRegion* getRegion(int x, int z, int layer);
+    [[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
-    WorldRegion* getOrCreateRegion(int x, int z, int layer);
+    [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord);
+    regfile_ptr createRegFile(glm::ivec2 coord);
+    void closeRegFile(glm::ivec2 coord);
 
-    /// @brief Compress buffer with extrle
+    WorldRegion* getRegion(int x, int z);
-    /// @param src source buffer
+    WorldRegion* getOrCreateRegion(int x, int z);
-    /// @param srclen length of the source buffer
-    /// @param len (out argument) length of result buffer
-    /// @return compressed bytes array
-    std::unique_ptr<ubyte[]> compress(
-        const ubyte* src, size_t srclen, size_t& len
-    );
 
-    /// @brief Decompress buffer with extrle
+    fs::path getRegionFilePath(int x, int z) const;
-    /// @param src compressed buffer
-    /// @param srclen length of compressed buffer
-    /// @param dstlen max expected length of source buffer
-    /// @return decompressed bytes array
-    std::unique_ptr<ubyte[]> decompress(
-        const ubyte* src, size_t srclen, size_t dstlen
-    );
 
-    std::unique_ptr<ubyte[]> readChunkData(
+    /// @brief Get chunk data. Read from file if not loaded yet.
-        int x, int y, uint32_t& length, regfile* file
+    /// @param x chunk x coord
-    );
+    /// @param z chunk z coord
-
+    /// @param size [out] compressed chunk data length
-    void fetchChunks(WorldRegion* region, int x, int y, regfile* file);
+    /// @param size [out] source chunk data length
-
+    /// @return nullptr if no saved chunk data found
-    ubyte* getData(int x, int z, int layer, uint32_t& size);
+    [[nodiscard]] ubyte* getData(int x, int z, uint32_t& size, uint32_t& srcSize);
-
-    regfile_ptr getRegFile(glm::ivec3 coord, bool create = true);
-    void closeRegFile(glm::ivec3 coord);
-    regfile_ptr useRegFile(glm::ivec3 coord);
-    regfile_ptr createRegFile(glm::ivec3 coord);
-
-    fs::path getRegionFilename(int x, int y) const;
-
-    void writeRegions(int layer);
 
     /// @brief Write or rewrite region file
     /// @param x region X
     /// @param z region Z
-    /// @param layer regions layer
+    void writeRegion(int x, int y, WorldRegion* entry);
-    void writeRegion(int x, int y, int layer, WorldRegion* entry);
+
+    /// @brief Write all unsaved regions to files
+    void writeAll();
+
+    /// @brief Read chunk data from region file
+    /// @param x chunk x coord
+    /// @param z chunk z coord
+    /// @param size [out] compressed chunk data length
+    /// @param srcSize [out] source chunk data length
+    /// @param rfile region file
+    /// @return nullptr if chunk is not present in region file
+    [[nodiscard]] static std::unique_ptr<ubyte[]> readChunkData(
+        int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile
+    );
+};
+
+class WorldRegions {
+    /// @brief World directory
+    fs::path directory;
+
+    RegionsLayer layers[REGION_LAYERS_COUNT] {};
 public:
     bool generatorTestMode = false;
     bool doWriteLights = true;
@@ -184,26 +198,57 @@ public:
     /// @param layer regions layer
     /// @param data target data
     /// @param size data size
-    /// @param rle compress with ext-RLE
     void put(
         int x,
         int z,
-        int layer,
+        RegionLayerIndex layer,
         std::unique_ptr<ubyte[]> data,
-        size_t size,
+        size_t size
-        bool rle
     );
 
-    std::unique_ptr<ubyte[]> getChunk(int x, int z);
+    /// @brief Get chunk voxels data
+    /// @param x chunk.x
+    /// @param z chunk.z
+    /// @return voxels data buffer or nullptr
+    std::unique_ptr<ubyte[]> getVoxels(int x, int z);
+
+    /// @brief Get cached lights for chunk at x,z
+    /// @return lights data or nullptr
     std::unique_ptr<light_t[]> getLights(int x, int z);
-    chunk_inventories_map fetchInventories(int x, int z);
+    
+    ChunkInventoriesMap fetchInventories(int x, int z);
+
+    BlocksMetadata getBlocksData(int x, int z);
+    
+    /// @brief Load saved entities data for chunk
+    /// @param x chunk.x
+    /// @param z chunk.z
+    /// @return map with entities list as "data"
     dv::value fetchEntities(int x, int z);
 
-    void processRegionVoxels(int x, int z, const regionproc& func);
+    /// @brief Load, process and save processed region chunks data
+    /// @param x region X
+    /// @param z region Z
+    /// @param layerid regions layer index
+    /// @param func processing callback
+    void processRegion(
+        int x, int z, RegionLayerIndex layerid, const RegionProc& func);
 
-    fs::path getRegionsFolder(int layer) const;
+    void processInventories(int x, int z, const InventoryProc& func);
 
-    void write();
+    void processBlocksData(int x, int z, const BlockDataProc& func);
+
+    /// @brief Get regions directory by layer index
+    /// @param layerid layer index
+    /// @return directory path
+    const fs::path& getRegionsFolder(RegionLayerIndex layerid) const;
+
+    fs::path getRegionFilePath(RegionLayerIndex layerid, int x, int z) const;
+
+    /// @brief Write all region layers
+    void writeAll();
+
+    void deleteRegion(RegionLayerIndex layerid, int x, int z);
 
     /// @brief Extract X and Z from 'X_Z.bin' region file name.
     /// @param name source region file name

src/files/compatibility.cpp

@@ -0,0 +1,110 @@
+#include "compatibility.hpp"
+
+#include <stdexcept>
+
+#include "constants.hpp"
+#include "voxels/voxel.hpp"
+#include "coders/compression.hpp"
+#include "coders/byte_utils.hpp"
+#include "lighting/Lightmap.hpp"
+#include "util/data_io.hpp"
+
+static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4;
+static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4;
+
+static util::Buffer<ubyte> convert_voxels_1to2(const ubyte* buffer, uint32_t size) {
+    auto data = compression::decompress(
+        buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8);
+
+    util::Buffer<ubyte> dstBuffer(VOXELS_DATA_SIZE_V2);
+    auto dst = reinterpret_cast<uint16_t*>(dstBuffer.data());
+
+    for (size_t i = 0; i < CHUNK_VOL; i++) {
+        ubyte bid1 = data[i];
+        ubyte bid2 = data[CHUNK_VOL + i];
+
+        ubyte bst1 = data[CHUNK_VOL * 2 + i];
+        ubyte bst2 = data[CHUNK_VOL * 3 + i];
+
+        dst[i] =
+            (static_cast<blockid_t>(bid1) << 8) | static_cast<blockid_t>(bid2);
+        dst[CHUNK_VOL + i] = (
+            (static_cast<blockstate_t>(bst1) << 8) |
+             static_cast<blockstate_t>(bst2)
+        );
+    }
+    size_t outLen;
+    auto compressed = compression::compress(
+        dstBuffer.data(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16);
+    return util::Buffer<ubyte>(std::move(compressed), outLen);
+}
+
+util::Buffer<ubyte> compatibility::convert_region_2to3(
+    const util::Buffer<ubyte>& src, RegionLayerIndex layer
+) {
+    const size_t REGION_CHUNKS = 1024;
+    const size_t HEADER_SIZE = 10;
+    const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t);
+    const ubyte COMPRESS_NONE = 0;
+    const ubyte COMPRESS_EXTRLE8 = 1;
+    const ubyte COMPRESS_EXTRLE16 = 2;
+
+    const ubyte* const ptr = src.data();
+
+    ByteBuilder builder;
+    builder.putCStr(".VOXREG");
+    builder.put(3);
+    switch (layer) {
+        case REGION_LAYER_VOXELS: builder.put(COMPRESS_EXTRLE16); break;
+        case REGION_LAYER_LIGHTS: builder.put(COMPRESS_EXTRLE8); break;
+        default: builder.put(COMPRESS_NONE); break;
+    }
+
+    uint32_t offsets[REGION_CHUNKS] {};
+    size_t chunkIndex = 0;
+
+    auto tablePtr = reinterpret_cast<const uint32_t*>(
+        ptr + src.size() - OFFSET_TABLE_SIZE
+    );
+
+    for (size_t i = 0; i < REGION_CHUNKS; i++) {
+        uint32_t srcOffset = dataio::be2h(tablePtr[i]);
+        if (srcOffset == 0) {
+            continue;
+        }
+        uint32_t size = *reinterpret_cast<const uint32_t*>(ptr + srcOffset);
+        size = dataio::be2h(size);
+
+        const ubyte* data = ptr + srcOffset + sizeof(uint32_t);
+        offsets[i] = builder.size();
+
+        switch (layer) {
+            case REGION_LAYER_VOXELS: {
+                auto dstdata = convert_voxels_1to2(data, size);
+                builder.putInt32(dstdata.size());
+                builder.putInt32(VOXELS_DATA_SIZE_V2);
+                builder.put(dstdata.data(), dstdata.size());
+                break;
+            }
+            case REGION_LAYER_LIGHTS:
+                builder.putInt32(size);
+                builder.putInt32(LIGHTMAP_DATA_LEN);
+                builder.put(data, size);
+                break;
+            case REGION_LAYER_ENTITIES:
+            case REGION_LAYER_INVENTORIES:
+            case REGION_LAYER_BLOCKS_DATA: {
+                builder.putInt32(size);
+                builder.putInt32(size);
+                builder.put(data, size);
+                break;
+            case REGION_LAYERS_COUNT: 
+                throw std::invalid_argument("invalid enum");
+            }
+        }
+    }
+    for (size_t i = 0; i < REGION_CHUNKS; i++) {
+        builder.putInt32(offsets[i]);
+    }
+    return util::Buffer<ubyte>(builder.build().data(), builder.size());
+}

src/files/compatibility.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "typedefs.hpp"
+#include "util/Buffer.hpp"
+#include "files/world_regions_fwd.hpp"
+
+namespace compatibility {
+    /// @brief Convert region file from version 2 to 3
+    /// @see /doc/specs/region_file_spec.md
+    /// @param src region file source content
+    /// @return new region file content
+    util::Buffer<ubyte> convert_region_2to3(
+        const util::Buffer<ubyte>& src, RegionLayerIndex layer);
+}

src/files/files.cpp

@@ -65,6 +65,12 @@ bool files::read(const fs::path& filename, char* data, size_t size) {
     return true;
 }
 
+util::Buffer<ubyte> files::read_bytes_buffer(const fs::path& path) {
+    size_t size;
+    auto bytes = files::read_bytes(path, size);
+    return util::Buffer<ubyte>(std::move(bytes), size);
+}
+
 std::unique_ptr<ubyte[]> files::read_bytes(
     const fs::path& filename, size_t& length
 ) {

src/files/files.hpp

@@ -8,6 +8,7 @@
 
 #include "typedefs.hpp"
 #include "data/dv.hpp"
+#include "util/Buffer.hpp"
 
 namespace fs = std::filesystem;
 
@@ -56,6 +57,7 @@ namespace files {
     );
 
     bool read(const fs::path&, char* data, size_t size);
+    util::Buffer<ubyte> read_bytes_buffer(const fs::path&);
     std::unique_ptr<ubyte[]> read_bytes(const fs::path&, size_t& length);
     std::vector<ubyte> read_bytes(const fs::path&);
     std::string read_string(const fs::path& filename);

src/files/world_regions_fwd.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "typedefs.hpp"
+
+enum RegionLayerIndex : uint {
+    REGION_LAYER_VOXELS = 0,
+    REGION_LAYER_LIGHTS,
+    REGION_LAYER_INVENTORIES,
+    REGION_LAYER_ENTITIES,
+    REGION_LAYER_BLOCKS_DATA,
+    
+    REGION_LAYERS_COUNT
+};

src/graphics/ui/gui_util.cpp

@@ -3,6 +3,7 @@
 #include "elements/Label.hpp"
 #include "elements/Menu.hpp"
 #include "elements/Button.hpp"
+#include "elements/TextBox.hpp"
 #include "gui_xml.hpp"
 
 #include "logic/scripting/scripting.hpp"
@@ -77,3 +78,47 @@ void guiutil::confirm(
     menu->addPage("<confirm>", panel);
     menu->setPage("<confirm>");
 }
+
+void guiutil::confirmWithMemo(
+        gui::GUI* gui, 
+        const std::wstring& text, 
+        const std::wstring& memo,
+        const runnable& on_confirm,
+        std::wstring yestext, 
+        std::wstring notext) {
+
+    if (yestext.empty()) yestext = langs::get(L"Yes");
+    if (notext.empty()) notext = langs::get(L"No");
+
+    auto menu = gui->getMenu();
+    auto panel = std::make_shared<Panel>(glm::vec2(600, 500), glm::vec4(8.0f), 8.0f);
+    panel->setColor(glm::vec4(0.0f, 0.0f, 0.0f, 0.5f));
+    panel->add(std::make_shared<Label>(text));
+    
+    auto textbox = std::make_shared<TextBox>(L"");
+    textbox->setMultiline(true);
+    textbox->setTextWrapping(true);
+    textbox->setSize(glm::vec2(600, 300));
+    textbox->setText(memo);
+    textbox->setEditable(false);
+    panel->add(textbox);
+
+    auto subpanel = std::make_shared<Panel>(glm::vec2(600, 53));
+    subpanel->setColor(glm::vec4(0));
+
+    subpanel->add(std::make_shared<Button>(yestext, glm::vec4(8.f), [=](GUI*){
+        if (on_confirm)
+            on_confirm();
+        menu->back();
+    }));
+
+    subpanel->add(std::make_shared<Button>(notext, glm::vec4(8.f), [=](GUI*){
+        menu->back();
+    }));
+
+    panel->add(subpanel);
+
+    panel->refresh();
+    menu->addPage("<confirm>", panel);
+    menu->setPage("<confirm>");
+}

src/graphics/ui/gui_util.hpp

@@ -24,4 +24,12 @@ namespace guiutil {
         const runnable& on_confirm=nullptr,
         std::wstring yestext=L"", 
         std::wstring notext=L"");
+
+    void confirmWithMemo(
+        gui::GUI* gui, 
+        const std::wstring& text, 
+        const std::wstring& memo,
+        const runnable& on_confirm=nullptr,
+        std::wstring yestext=L"", 
+        std::wstring notext=L"");
 }

src/items/Inventory.cpp

@@ -1,6 +1,6 @@
 #include "Inventory.hpp"
 
-#include "content/ContentLUT.hpp"
+#include "content/ContentReport.hpp"
 
 Inventory::Inventory(int64_t id, size_t size) : id(id), slots(size) {
 }
@@ -83,11 +83,20 @@ dv::value Inventory::serialize() const {
     return map;
 }
 
-void Inventory::convert(dv::value& data, const ContentLUT* lut) {
+void Inventory::convert(const ContentReport* report) {
+    for (auto& slot : slots) {
+        itemid_t id = slot.getItemId();
+        itemid_t replacement = report->items.getId(id);
+        slot.set(ItemStack(replacement, slot.getCount()));
+    }
+}
+
+// TODO: remove
+void Inventory::convert(dv::value& data, const ContentReport* report) {
     auto& slotsarr = data["slots"];
     for (auto& item : data["slots"]) {
         itemid_t id = item["id"].asInteger(ITEM_EMPTY);
-        itemid_t replacement = lut->items.getId(id);
+        itemid_t replacement = report->items.getId(id);
         item["id"] = replacement;
         if (replacement == 0 && item.has("count")) {
             item.erase("count");

src/items/Inventory.hpp

@@ -7,7 +7,7 @@
 #include "typedefs.hpp"
 #include "ItemStack.hpp"
 
-class ContentLUT;
+class ContentReport;
 class ContentIndices;
 
 class Inventory : public Serializable {
@@ -37,7 +37,8 @@ public:
 
     dv::value serialize() const override;
 
-    static void convert(dv::value& data, const ContentLUT* lut);
+    void convert(const ContentReport* report);
+    static void convert(dv::value& data, const ContentReport* report);
 
     inline void setId(int64_t id) {
         this->id = id;

src/items/ItemStack.cpp

@@ -16,6 +16,9 @@ void ItemStack::set(const ItemStack& item) {
     if (count == 0) {
         this->item = 0;
     }
+    if (this->item == 0) {
+        count = 0;
+    }
 }
 
 bool ItemStack::accepts(const ItemStack& other) const {

src/logic/EngineController.cpp

@@ -4,10 +4,10 @@
 #include <filesystem>
 #include <memory>
 
-#include "coders/commons.hpp"
-#include "content/ContentLUT.hpp"
-#include "debug/Logger.hpp"
 #include "engine.hpp"
+#include "coders/commons.hpp"
+#include "debug/Logger.hpp"
+#include "content/ContentReport.hpp"
 #include "files/WorldConverter.hpp"
 #include "files/WorldFiles.hpp"
 #include "frontend/locale.hpp"
@@ -46,19 +46,28 @@ std::shared_ptr<Task> create_converter(
     Engine* engine,
     const std::shared_ptr<WorldFiles>& worldFiles,
     const Content* content,
-    const std::shared_ptr<ContentLUT>& lut,
+    const std::shared_ptr<ContentReport>& report,
     const runnable& postRunnable
 ) {
+    ConvertMode mode;
+    if (report->isUpgradeRequired()) {
+        mode = ConvertMode::UPGRADE;
+    } else if (report->hasContentReorder()) {
+        mode = ConvertMode::REINDEX;
+    } else {
+        mode = ConvertMode::BLOCK_FIELDS;
+    }
     return WorldConverter::startTask(
         worldFiles,
         content,
-        lut,
+        report,
         [=]() {
             auto menu = engine->getGUI()->getMenu();
             menu->reset();
             menu->setPage("main", false);
             engine->getGUI()->postRunnable([=]() { postRunnable(); });
         },
+        mode,
         true
     );
 }
@@ -66,31 +75,55 @@ std::shared_ptr<Task> create_converter(
 void show_convert_request(
     Engine* engine,
     const Content* content,
-    const std::shared_ptr<ContentLUT>& lut,
+    const std::shared_ptr<ContentReport>& report,
     const std::shared_ptr<WorldFiles>& worldFiles,
     const runnable& postRunnable
 ) {
-    guiutil::confirm(
+    auto on_confirm = [=]() {
-        engine->getGUI(),
-        langs::get(L"world.convert-request"),
-        [=]() {
             auto converter =
-                create_converter(engine, worldFiles, content, lut, postRunnable);
+                create_converter(engine, worldFiles, content, report, postRunnable);
             menus::show_process_panel(
                 engine, converter, L"Converting world..."
             );
-        },
+        };
+
+    std::wstring message = L"world.convert-block-layouts";
+    if (report->hasContentReorder()) {
+        message = L"world.convert-request";
+    }
+    if (report->isUpgradeRequired()) {
+        message = L"world.upgrade-request";
+    } else if (report->hasDataLoss()) {
+        message = L"world.convert-with-loss";
+        std::wstring text;
+        for (const auto& line : report->getDataLoss()) {
+            text += util::str2wstr_utf8(line) + L"\n";
+        }
+        guiutil::confirmWithMemo(
+            engine->getGUI(),
+            langs::get(message),
+            text,
+            on_confirm,
+            L"",
+            langs::get(L"Cancel")
+        );
+        return;
+    }
+    guiutil::confirm(
+        engine->getGUI(),
+        langs::get(message),
+        on_confirm,
         L"",
         langs::get(L"Cancel")
     );
 }
 
 static void show_content_missing(
-    Engine* engine, const std::shared_ptr<ContentLUT>& lut
+    Engine* engine, const std::shared_ptr<ContentReport>& report
 ) {
     auto root = dv::object();
     auto& contentEntries = root.list("content");
-    for (auto& entry : lut->getMissingContent()) {
+    for (auto& entry : report->getMissingContent()) {
         std::string contentName = ContentType_name(entry.type);
         auto& contentEntry = contentEntries.object();
         contentEntry["type"] = contentName;
@@ -134,10 +167,10 @@ void EngineController::openWorld(const std::string& name, bool confirmConvert) {
     auto* content = engine->getContent();
     auto worldFiles = std::make_shared<WorldFiles>(
         folder, engine->getSettings().debug);
-    if (auto lut = World::checkIndices(worldFiles, content)) {
+    if (auto report = World::checkIndices(worldFiles, content)) {
-        if (lut->hasMissingContent()) {
+        if (report->hasMissingContent()) {
             engine->setScreen(std::make_shared<MenuScreen>(engine));
-            show_content_missing(engine, lut);
+            show_content_missing(engine, report);
         } else {
             if (confirmConvert) {
                 menus::show_process_panel(
@@ -146,13 +179,13 @@ void EngineController::openWorld(const std::string& name, bool confirmConvert) {
                         engine,
                         worldFiles,
                         content,
-                        lut,
+                        report,
                         [=]() { openWorld(name, false); }
                     ),
                     L"Converting world..."
                 );
             } else {
-                show_convert_request(engine, content, lut, std::move(worldFiles), [=]() {
+                show_convert_request(engine, content, report, std::move(worldFiles), [=]() {
                     openWorld(name, false);
                 });
             }

src/logic/scripting/lua/libblock.cpp

@@ -7,6 +7,8 @@
 #include "voxels/Chunks.hpp"
 #include "voxels/voxel.hpp"
 #include "world/Level.hpp"
+#include "maths/voxmaths.hpp"
+#include "data/StructLayout.hpp"
 #include "api_lua.hpp"
 
 using namespace scripting;
@@ -436,6 +438,134 @@ static int l_decompose_state(lua::State* L) {
     return 1;
 }
 
+static int get_field(
+    lua::State* L,
+    const ubyte* src,
+    const data::Field& field,
+    size_t index,
+    const data::StructLayout& dataStruct
+) {
+    switch (field.type) {
+        case data::FieldType::I8:
+        case data::FieldType::I16:
+        case data::FieldType::I32:
+        case data::FieldType::I64:
+            return lua::pushinteger(L, dataStruct.getInteger(src, field, index));
+        case data::FieldType::F32:
+        case data::FieldType::F64:
+            return lua::pushnumber(L, dataStruct.getNumber(src, field, index));
+        case data::FieldType::CHAR:
+            return lua::pushstring(L, 
+                std::string(dataStruct.getChars(src, field)).c_str());
+    }
+    return 0;
+}
+
+static int l_get_field(lua::State* L) {
+    auto x = lua::tointeger(L, 1);
+    auto y = lua::tointeger(L, 2);
+    auto z = lua::tointeger(L, 3);
+    auto name = lua::require_string(L, 4);
+    size_t index = 0;
+    if (lua::gettop(L) >= 5) {
+        index = lua::tointeger(L, 5);
+    }
+    auto vox = level->chunks->get(x, y, z);
+    auto cx = floordiv(x, CHUNK_W);
+    auto cz = floordiv(z, CHUNK_D);
+    auto chunk = level->chunks->getChunk(cx, cz);
+    auto lx = x - cx * CHUNK_W;
+    auto lz = z - cz * CHUNK_W;
+    size_t voxelIndex = vox_index(lx, y, lz);
+
+    const auto& def = content->getIndices()->blocks.require(vox->id);
+    if (def.dataStruct == nullptr) {
+        return 0;
+    }
+    const auto& dataStruct = *def.dataStruct;
+    const auto field = dataStruct.getField(name);
+    if (field == nullptr) {
+        return 0;
+    }
+    if (index >= field->elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field->elements)+"]");
+    }
+    const ubyte* src = chunk->blocksMetadata.find(voxelIndex);
+    if (src == nullptr) {
+        return 0;
+    }
+    return get_field(L, src, *field, index, dataStruct);
+}
+
+static int set_field(
+    lua::State* L,
+    ubyte* dst,
+    const data::Field& field,
+    size_t index,
+    const data::StructLayout& dataStruct,
+    const dv::value& value
+) {
+    switch (field.type) {
+        case data::FieldType::CHAR:
+            if (value.isString()) {
+                return lua::pushinteger(L,
+                    dataStruct.setUnicode(dst, value.asString(), field));
+            }
+        case data::FieldType::I8:
+        case data::FieldType::I16:
+        case data::FieldType::I32:
+        case data::FieldType::I64:
+            dataStruct.setInteger(dst, value.asInteger(), field, index);
+            break;
+        case data::FieldType::F32:
+        case data::FieldType::F64:
+            dataStruct.setNumber(dst, value.asNumber(), field, index);
+            break;
+    }
+    return 0;
+}
+
+static int l_set_field(lua::State* L) {
+    auto x = lua::tointeger(L, 1);
+    auto y = lua::tointeger(L, 2);
+    auto z = lua::tointeger(L, 3);
+    auto name = lua::require_string(L, 4);
+    auto value = lua::tovalue(L, 5);
+    size_t index = 0;
+    if (lua::gettop(L) >= 6) {
+        index = lua::tointeger(L, 6);
+    }
+    auto vox = level->chunks->get(x, y, z);
+    auto cx = floordiv(x, CHUNK_W);
+    auto cz = floordiv(z, CHUNK_D);
+    auto chunk = level->chunks->getChunk(cx, cz);
+    auto lx = x - cx * CHUNK_W;
+    auto lz = z - cz * CHUNK_W;
+    size_t voxelIndex = vox_index(lx, y, lz);
+
+    const auto& def = content->getIndices()->blocks.require(vox->id);
+    if (def.dataStruct == nullptr) {
+        return 0;
+    }
+    const auto& dataStruct = *def.dataStruct;
+    const auto field = dataStruct.getField(name);
+    if (field == nullptr) {
+        return 0;
+    }
+    if (index >= field->elements) {
+        throw std::out_of_range(
+            "index out of bounds [0, "+std::to_string(field->elements)+"]");
+    }
+    ubyte* dst = chunk->blocksMetadata.find(voxelIndex);
+    if (dst == nullptr) {
+        dst = chunk->blocksMetadata.allocate(voxelIndex, dataStruct.size());
+    }
+    chunk->flags.unsaved = true;
+    chunk->flags.blocksData = true;
+    return set_field(L, dst, *field, index, dataStruct, value);
+}
+
 const luaL_Reg blocklib[] = {
     {"index", lua::wrap<l_index>},
     {"name", lua::wrap<l_get_def>},
@@ -469,5 +599,7 @@ const luaL_Reg blocklib[] = {
     {"raycast", lua::wrap<l_raycast>},
     {"compose_state", lua::wrap<l_compose_state>},
     {"decompose_state", lua::wrap<l_decompose_state>},
+    {"get_field", lua::wrap<l_get_field>},
+    {"set_field", lua::wrap<l_set_field>},
     {NULL, NULL}
 };

src/logic/scripting/lua/lua_util.cpp

@@ -51,12 +51,23 @@ int lua::pushvalue(State* L, const dv::value& value) {
             }
             break;
         }
-        case value_type::object: {
+        case value_type::object:
             createtable(L, 0, value.size());
             for (const auto& [key, elem] : value.asObject()) {
                 pushvalue(L, elem);
                 setfield(L, key);
             }
+            break;
+        case value_type::bytes: {
+            const auto& bytes = value.asBytes();
+            createtable(L, 0, bytes.size());
+            size_t size = bytes.size();
+            for (size_t i = 0; i < size;) {
+                pushinteger(L, bytes[i]);
+                i++;
+                rawseti(L, i);
+            }
+            break;
         }
     }
     return 1;

src/objects/Player.cpp

@@ -4,7 +4,7 @@
 #include <glm/glm.hpp>
 #include <utility>
 
-#include "content/ContentLUT.hpp"
+#include "content/ContentReport.hpp"
 #include "items/Inventory.hpp"
 #include "Entities.hpp"
 #include "rigging.hpp"
@@ -308,17 +308,17 @@ void Player::deserialize(const dv::value& src) {
     }
 }
 
-void Player::convert(dv::value& data, const ContentLUT* lut) {
+void Player::convert(dv::value& data, const ContentReport* report) {
     if (data.has("players")) {
         auto& players = data["players"];
         for (uint i = 0; i < players.size(); i++) {
             auto& playerData = players[i];
             if (playerData.has("inventory")) {
-                Inventory::convert(playerData["inventory"], lut);
+                Inventory::convert(playerData["inventory"], report);
             }
         }
 
     } else if (data.has("inventory")){
-        Inventory::convert(data["inventory"], lut);
+        Inventory::convert(data["inventory"], report);
     }
 }

src/objects/Player.hpp

@@ -11,7 +11,7 @@
 
 class Camera;
 class Inventory;
-class ContentLUT;
+class ContentReport;
 class Level;
 struct Hitbox;
 struct EngineSettings;
@@ -105,7 +105,7 @@ public:
     dv::value serialize() const override;
     void deserialize(const dv::value& src) override;
 
-    static void convert(dv::value& data, const ContentLUT* lut);
+    static void convert(dv::value& data, const ContentReport* report);
 
     inline int getId() const {
         return objectUID;

src/util/Buffer.hpp

@@ -4,6 +4,8 @@
 #include <cstring>
 
 namespace util {
+    /// @brief Template similar to std::unique_ptr stores a buffer with its size
+    /// @tparam T buffer elements type
     template<typename T>
     class Buffer {
         std::unique_ptr<T[]> ptr;
@@ -12,7 +14,9 @@ namespace util {
         Buffer(size_t length)
          : ptr(std::make_unique<T[]>(length)), length(length) {
         }
-        Buffer(const Buffer<T>& o) : Buffer(o.data(), o.size()) {}
+        explicit Buffer(const Buffer<T>& o) : Buffer(o.data(), o.size()) {}
+
+        Buffer(Buffer<T>&& o) : ptr(std::move(o.ptr)), length(o.length) {}
 
         Buffer(std::unique_ptr<T[]> ptr, size_t length)
          : ptr(std::move(ptr)), length(length) {}
@@ -42,16 +46,28 @@ namespace util {
             return length;
         }
 
+        /// @brief Take ownership over the buffer unique_ptr
         std::unique_ptr<T[]> release() {
             return std::move(ptr);
         }
 
+        /// @brief Create a buffer copy 
         Buffer clone() const {
             return Buffer(ptr.get(), length);
         }
 
+        /// @brief Update buffer size without releasing used memory
+        /// @param size new size (must be less or equal to current)
         void resizeFast(size_t size) {
             length = size;
         }
+
+        const T* begin() const {
+            return ptr.get();
+        }
+
+        const T* end() const {
+            return ptr.get() + length;
+        }
     };
 }

src/util/BufferPool.hpp

@@ -35,5 +35,9 @@ namespace util {
                 freeBuffers.push(ptr);
             });
         }
+
+        size_t getBufferSize() const {
+            return bufferSize;
+        }
     };
 }

src/util/SmallHeap.hpp

@@ -0,0 +1,215 @@
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <vector>
+#include <limits>
+#include <stdexcept>
+
+#include "Buffer.hpp"
+#include "data_io.hpp"
+
+namespace util {    
+    template<typename T>
+    inline T read_int_le(const uint8_t* src, size_t offset=0) {
+        return dataio::le2h(*(reinterpret_cast<const T*>(src) + offset));
+    }
+
+    /// @brief Simple heap implementation for memory-optimal sparse array of 
+    /// small different structures
+    /// @note alignment is not impemented 
+    /// (impractical in the context of scripting and memory consumption)
+    /// @tparam Tindex entry index type
+    /// @tparam Tsize entry size type
+    template <typename Tindex, typename Tsize>
+    class SmallHeap {
+        std::vector<uint8_t> buffer;
+        Tindex entriesCount;
+    public:
+        SmallHeap() : entriesCount(0) {}
+
+        /// @brief Find current entry address by index
+        /// @param index entry index
+        /// @return temporary raw pointer or nullptr if entry does not exists
+        /// @attention pointer becomes invalid after allocate(...) or free(...)
+        uint8_t* find(Tindex index) {
+            auto data = buffer.data();
+            for (size_t i = 0; i < entriesCount; i++) {
+                auto nextIndex = read_int_le<Tindex>(data);
+                data += sizeof(Tindex);
+                auto nextSize = read_int_le<Tsize>(data);
+                data += sizeof(Tsize);
+                if (nextIndex == index) {
+                    return data;
+                } else if (nextIndex > index) {
+                    return nullptr;
+                }
+                data += nextSize;
+            }
+            return nullptr;
+        }
+
+        /// @brief Erase entry from the heap
+        /// @param ptr valid entry pointer
+        void free(uint8_t* ptr) {
+            if (ptr == nullptr) {
+                return;
+            }
+            auto entrySize = sizeOf(ptr);
+            auto begin =
+                buffer.begin() +
+                ((ptr - sizeof(Tsize) - sizeof(Tindex)) - buffer.data());
+            buffer.erase(
+                begin, begin + entrySize + sizeof(Tsize) + sizeof(Tindex)
+            );
+            entriesCount--;
+        }
+
+        /// @brief Create or update entry (size)
+        /// @param index entry index
+        /// @param size entry size
+        /// @return temporary entry pointer 
+        /// @attention pointer becomes invalid after allocate(...) or free(...)
+        uint8_t* allocate(Tindex index, size_t size) {
+            const auto maxSize = std::numeric_limits<Tsize>::max();
+            if (size > maxSize) {
+                throw std::invalid_argument(
+                    "requested "+std::to_string(size)+" bytes but limit is "+
+                    std::to_string(maxSize));
+            }
+            if (size == 0) {
+                throw std::invalid_argument("zero size");
+            }
+            ptrdiff_t offset = 0;
+            if (auto found = find(index)) {
+                auto entrySize = sizeOf(found);
+                if (size == entrySize) {
+                    std::memset(found, 0, entrySize);
+                    return found;
+                }
+                this->free(found);
+                return allocate(index, size);
+            }
+            for (size_t i = 0; i < entriesCount; i++) {
+                auto data = buffer.data() + offset;
+                auto nextIndex = read_int_le<Tindex>(data);
+                data += sizeof(Tindex);
+                auto nextSize = read_int_le<Tsize>(data);
+                data += sizeof(Tsize);
+                if (nextIndex > index) {
+                    break;
+                }
+                data += nextSize;
+                offset = data - buffer.data();
+            }
+            buffer.insert(
+                buffer.begin() + offset,
+                size + sizeof(Tindex) + sizeof(Tsize),
+                0
+            );
+            entriesCount++;
+
+            auto data = buffer.data() + offset;
+            *reinterpret_cast<Tindex*>(data) = dataio::h2le(index);
+            data += sizeof(Tindex);
+            *reinterpret_cast<Tsize*>(data) = dataio::h2le(size);
+            return data + sizeof(Tsize);
+        }
+
+        /// @param ptr valid entry pointer
+        /// @return entry size
+        Tsize sizeOf(uint8_t* ptr) const {
+            if (ptr == nullptr) {
+                return 0;
+            }
+            return read_int_le<Tsize>(ptr, -1);
+        }
+
+        /// @return number of entries
+        Tindex count() const {
+            return entriesCount;
+        }
+
+        /// @return total used bytes including entries metadata
+        size_t size() const {
+            return buffer.size();
+        }
+
+        inline bool operator==(const SmallHeap<Tindex, Tsize>& o) const {
+            if (o.entriesCount != entriesCount) {
+                return false;
+            }
+            return buffer == o.buffer;
+        }
+
+        util::Buffer<uint8_t> serialize() const {
+            util::Buffer<uint8_t> out(sizeof(Tindex) + buffer.size());
+            ubyte* dst = out.data();
+            const ubyte* src = buffer.data();
+
+            *reinterpret_cast<Tindex*>(dst) = dataio::h2le(entriesCount);
+            dst += sizeof(Tindex);
+
+            std::memcpy(dst, src, buffer.size());
+            return out;
+        }
+
+        void deserialize(const ubyte* src, size_t size) {
+            entriesCount = read_int_le<Tindex>(src);
+            buffer.resize(size - sizeof(Tindex));
+            std::memcpy(buffer.data(), src + sizeof(Tindex), buffer.size());
+        }
+
+        struct const_iterator {
+        private:
+            const std::vector<uint8_t>& buffer;
+        public:
+            Tindex index;
+            size_t offset;
+
+            const_iterator(
+                const std::vector<uint8_t>& buffer, Tindex index, size_t offset
+            ) : buffer(buffer), index(index), offset(offset) {}
+
+            Tsize size() const {
+                return read_int_le<Tsize>(buffer.data() + offset, -1);
+            }
+
+            bool operator!=(const const_iterator& o) const {
+                return o.offset != offset;
+            }
+
+            const_iterator& operator++() {
+                offset += size();
+                if (offset == buffer.size()) {
+                    return *this;
+                }
+                index = read_int_le<Tindex>(buffer.data() + offset);
+                offset += sizeof(Tindex) + sizeof(Tsize);
+                return *this;
+            }
+
+            const_iterator& operator*() {
+                return *this;
+            }
+
+            const uint8_t* data() const {
+                return buffer.data() + offset;
+            }
+        };
+
+        const_iterator begin() const {
+            if (buffer.empty()) {
+                return end();
+            }
+            return const_iterator (
+                buffer,
+                read_int_le<Tindex>(buffer.data()),
+                sizeof(Tindex) + sizeof(Tsize));
+        }
+
+        const_iterator end() const {
+            return const_iterator (buffer, 0, buffer.size());
+        }
+    };
+}

src/util/data_io.hpp

@@ -3,16 +3,85 @@
 #include "typedefs.hpp"
 
 namespace dataio {
-    /* Read big-endian 16 bit signed integer from bytes */
+    /// @brief Swap byte-order for value of type T
+    /// @tparam T value type
+    /// @param value source integer
+    /// @return swapped integer
+    template <typename T>
+    T swap(T value) {
+        union{
+            T value;
+            ubyte bytes[sizeof(T)];
+        } source, dest;
+
+        source.value = value;
+
+        for (size_t i = 0; i < sizeof(T); i++) {
+            dest.bytes[i] = source.bytes[sizeof(T) - i - 1];
+        }
+        return dest.value;
+    }
+
+    inline bool is_big_endian() {
+        uint16_t num = 1;
+        auto bytes = reinterpret_cast<uint8_t*>(&num);
+        return bytes[1] == 1;
+    }
+
+    /// @brief Convert big-endian to hardware byte-order
+    /// @tparam T value type
+    /// @param value source integer
+    /// @return integer with hardware byte-order
+    template <typename T>
+    inline T be2h(T value){
+        if (is_big_endian()) {
+            return value;
+        } else {
+            return swap(value);
+        }
+    }
+
+    /// @brief Convert hardware byte-order to big-endian 
+    /// @tparam T value type
+    /// @param value source integer
+    /// @return big-endian integer
+    template <typename T>
+    T h2be(T value){
+        return be2h(value);
+    }
+
+    /// @brief Convert little-endian to hardware byte-order
+    /// @tparam T value type
+    /// @param value source integer
+    /// @return integer with hardware byte-order
+    template <typename T>
+    T le2h(T value){
+        if (is_big_endian()) {
+            return swap(value);
+        } else {
+            return value;
+        }
+    }
+
+    /// @brief Convert hardware byte-order to little-endian 
+    /// @tparam T value type
+    /// @param value source integer
+    /// @return little-endian integer
+    template <typename T>
+    T h2le(T value){
+        return le2h(value);
+    }
+
+    /// @brief Read big-endian 16 bit signed integer from bytes
     inline int16_t read_int16_big(const ubyte* src, size_t offset) {
         return (src[offset] << 8) | (src[offset + 1]);
     }
-    /* Read big-endian 32 bit signed integer from bytes */
+    /// @brief Read big-endian 32 bit signed integer from bytes
     inline int32_t read_int32_big(const ubyte* src, size_t offset) {
         return (src[offset] << 24) | (src[offset + 1] << 16) |
                (src[offset + 2] << 8) | (src[offset + 3]);
     }
-    /* Read big-endian 64 bit signed integer from bytes */
+    /// @brief Read big-endian 64 bit signed integer from bytes
     inline int64_t read_int64_big(const ubyte* src, size_t offset) {
         return (int64_t(src[offset]) << 56) | (int64_t(src[offset + 1]) << 48) |
                (int64_t(src[offset + 2]) << 40) |
@@ -21,19 +90,19 @@ namespace dataio {
                (int64_t(src[offset + 5]) << 16) |
                (int64_t(src[offset + 6]) << 8) | (int64_t(src[offset + 7]));
     }
-    /* Write big-endian 16 bit signed integer to bytes */
+    /// @brief Write big-endian 16 bit signed integer to bytes
     inline void write_int16_big(int16_t value, ubyte* dest, size_t offset) {
         dest[offset] = (char)(value >> 8 & 255);
         dest[offset + 1] = (char)(value >> 0 & 255);
     }
-    /* Write big-endian 32 bit signed integer to bytes */
+    /// @brief Write big-endian 32 bit signed integer to bytes
     inline void write_int32_big(int32_t value, ubyte* dest, size_t offset) {
         dest[offset] = (char)(value >> 24 & 255);
         dest[offset + 1] = (char)(value >> 16 & 255);
         dest[offset + 2] = (char)(value >> 8 & 255);
         dest[offset + 3] = (char)(value >> 0 & 255);
     }
-    /* Write big-endian 64 bit signed integer to bytes */
+    /// @brief Write big-endian 64 bit signed integer to bytes
     inline void write_int64_big(int64_t value, ubyte* dest, size_t offset) {
         dest[offset] = (char)(value >> 56 & 255);
         dest[offset + 1] = (char)(value >> 48 & 255);

src/voxels/Block.cpp

@@ -1,8 +1,10 @@
 #include "Block.hpp"
 
+#include <set>
 #include <utility>
 
 #include "core_defs.hpp"
+#include "data/StructLayout.hpp"
 #include "util/stringutil.hpp"
 
 std::string to_string(BlockModel model) {
@@ -102,6 +104,8 @@ Block::Block(const std::string& name)
       } {
 }
 
+Block::~Block() {}
+
 Block::Block(std::string name, const std::string& texture)
     : name(std::move(name)),
       textureFaces {texture, texture, texture, texture, texture, texture} {
@@ -138,3 +142,10 @@ void Block::cloneTo(Block& dst) {
     dst.inventorySize = inventorySize;
     dst.tickInterval = tickInterval;
 }
+
+static std::set<std::string, std::less<>> RESERVED_BLOCK_FIELDS {
+};
+
+bool Block::isReservedBlockField(std::string_view view) {
+    return RESERVED_BLOCK_FIELDS.find(view) != RESERVED_BLOCK_FIELDS.end();
+}

src/voxels/Block.hpp

@@ -9,6 +9,10 @@
 #include "maths/aabb.hpp"
 #include "typedefs.hpp"
 
+namespace data {
+    class StructLayout;
+}
+
 inline std::string BLOCK_ITEM_SUFFIX = ".item";
 
 inline constexpr uint FACE_MX = 0;
@@ -22,6 +26,8 @@ inline constexpr uint FACE_PZ = 5;
 /// complex hitboxes
 inline constexpr uint BLOCK_AABB_GRID = 16;
 
+inline constexpr size_t MAX_USER_BLOCK_FIELDS_SIZE = 240;
+
 inline std::string DEFAULT_MATERIAL = "base:stone";
 
 struct block_funcs_set {
@@ -184,6 +190,8 @@ public:
     // @brief Block tick interval (1 - 20tps, 2 - 10tps)
     uint tickInterval = 1;
 
+    std::unique_ptr<data::StructLayout> dataStruct;
+
     /// @brief Runtime indices (content indexing results)
     struct {
         /// @brief block runtime integer id
@@ -211,8 +219,11 @@ public:
     Block(const std::string& name);
     Block(std::string name, const std::string& texture);
     Block(const Block&) = delete;
+    ~Block();
 
     void cloneTo(Block& dst);
+
+    static bool isReservedBlockField(std::string_view view);
 };
 
 inline glm::ivec3 get_ground_direction(const Block& def, int rotation) {

src/voxels/Chunk.cpp

@@ -2,9 +2,10 @@
 
 #include <utility>
 
-#include "content/ContentLUT.hpp"
+#include "content/ContentReport.hpp"
 #include "items/Inventory.hpp"
 #include "lighting/Lightmap.hpp"
+#include "util/data_io.hpp"
 #include "voxel.hpp"
 
 Chunk::Chunk(int xpos, int zpos) : x(xpos), z(zpos) {
@@ -53,7 +54,7 @@ void Chunk::removeBlockInventory(uint x, uint y, uint z) {
     }
 }
 
-void Chunk::setBlockInventories(chunk_inventories_map map) {
+void Chunk::setBlockInventories(ChunkInventoriesMap map) {
     inventories = std::move(map);
 }
 
@@ -78,59 +79,41 @@ std::unique_ptr<Chunk> Chunk::clone() const {
 
 /**
   Current chunk format:
-    - byte-order: big-endian
+    - byte-order: little-endian
-    - [don't panic!] first and second bytes are separated for RLE efficiency
 
     ```cpp
-    uint8_t voxel_id_first_byte[CHUNK_VOL];
+    uint16_t voxel_id[CHUNK_VOL];
-    uint8_t voxel_id_second_byte[CHUNK_VOL];
+    uint16_t voxel_states[CHUNK_VOL];
-    uint8_t voxel_states_first_byte[CHUNK_VOL];
-    uint8_t voxel_states_second_byte[CHUNK_VOL];
     ```
 
     Total size: (CHUNK_VOL * 4) bytes
 */
 std::unique_ptr<ubyte[]> Chunk::encode() const {
     auto buffer = std::make_unique<ubyte[]>(CHUNK_DATA_LEN);
+    auto dst = reinterpret_cast<uint16_t*>(buffer.get());
     for (uint i = 0; i < CHUNK_VOL; i++) {
-        buffer[i] = voxels[i].id >> 8;
+        dst[i] = dataio::h2le(voxels[i].id);
-        buffer[CHUNK_VOL + i] = voxels[i].id & 0xFF;
+        dst[CHUNK_VOL + i] = dataio::h2le(blockstate2int(voxels[i].state));
-
-        blockstate_t state = blockstate2int(voxels[i].state);
-        buffer[CHUNK_VOL * 2 + i] = state >> 8;
-        buffer[CHUNK_VOL * 3 + i] = state & 0xFF;
     }
     return buffer;
 }
 
 bool Chunk::decode(const ubyte* data) {
+    auto src = reinterpret_cast<const uint16_t*>(data);
     for (uint i = 0; i < CHUNK_VOL; i++) {
         voxel& vox = voxels[i];
 
-        ubyte bid1 = data[i];
+        vox.id = dataio::le2h(src[i]);
-        ubyte bid2 = data[CHUNK_VOL + i];
+        vox.state = int2blockstate(dataio::le2h(src[CHUNK_VOL + i]));
-
-        ubyte bst1 = data[CHUNK_VOL * 2 + i];
-        ubyte bst2 = data[CHUNK_VOL * 3 + i];
-
-        vox.id =
-            (static_cast<blockid_t>(bid1) << 8) | static_cast<blockid_t>(bid2);
-        vox.state = int2blockstate(
-            (static_cast<blockstate_t>(bst1) << 8) |
-            static_cast<blockstate_t>(bst2)
-        );
     }
     return true;
 }
 
-void Chunk::convert(ubyte* data, const ContentLUT* lut) {
+void Chunk::convert(ubyte* data, const ContentReport* report) {
+    auto buffer = reinterpret_cast<uint16_t*>(data);
     for (uint i = 0; i < CHUNK_VOL; i++) {
-        // see encode method to understand what the hell is going on here
+        blockid_t id = dataio::le2h(buffer[i]);
-        blockid_t id =
+        blockid_t replacement = report->blocks.getId(id);
-            ((static_cast<blockid_t>(data[i]) << 8) |
+        buffer[i] = dataio::h2le(replacement);
-             static_cast<blockid_t>(data[CHUNK_VOL + i]));
-        blockid_t replacement = lut->blocks.getId(id);
-        data[i] = replacement >> 8;
-        data[CHUNK_VOL + i] = replacement & 0xFF;
     }
 }

src/voxels/Chunk.hpp

@@ -7,17 +7,19 @@
 
 #include "constants.hpp"
 #include "lighting/Lightmap.hpp"
+#include "util/SmallHeap.hpp"
 #include "voxel.hpp"
 
 inline constexpr int CHUNK_DATA_LEN = CHUNK_VOL * 4;
 
-class Lightmap;
+class ContentReport;
-class ContentLUT;
 class Inventory;
 
-using chunk_inventories_map =
+using ChunkInventoriesMap =
     std::unordered_map<uint, std::shared_ptr<Inventory>>;
 
+using BlocksMetadata = util::SmallHeap<uint16_t, uint8_t>;
+
 class Chunk {
 public:
     int x, z;
@@ -32,10 +34,13 @@ public:
         bool unsaved : 1;
         bool loadedLights : 1;
         bool entities : 1;
+        bool blocksData : 1;
     } flags {};
 
     /// @brief Block inventories map where key is index of block in voxels array
-    chunk_inventories_map inventories;
+    ChunkInventoriesMap inventories;
+    /// @brief Blocks metadata heap
+    BlocksMetadata blocksMetadata;
 
     Chunk(int x, int z);
 
@@ -52,7 +57,7 @@ public:
         std::shared_ptr<Inventory> inventory, uint x, uint y, uint z
     );
     void removeBlockInventory(uint x, uint y, uint z);
-    void setBlockInventories(chunk_inventories_map map);
+    void setBlockInventories(ChunkInventoriesMap map);
 
     /// @return inventory bound to the given block or nullptr
     std::shared_ptr<Inventory> getBlockInventory(uint x, uint y, uint z) const;
@@ -62,10 +67,12 @@ public:
         flags.unsaved = true;
     }
 
+    /// @brief Encode chunk to bytes array of size CHUNK_DATA_LEN
+    /// @see /doc/specs/region_voxels_chunk_spec.md
     std::unique_ptr<ubyte[]> encode() const;
 
     /// @return true if all is fine
     bool decode(const ubyte* data);
 
-    static void convert(ubyte* data, const ContentLUT* lut);
+    static void convert(ubyte* data, const ContentReport* report);
 };

src/voxels/Chunks.cpp

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <vector>
 
+#include "data/StructLayout.hpp"
 #include "coders/byte_utils.hpp"
 #include "coders/json.hpp"
 #include "content/Content.hpp"
@@ -363,6 +364,7 @@ void Chunks::set(
     }
     int lx = x - cx * CHUNK_W;
     int lz = z - cz * CHUNK_D;
+    size_t index = vox_index(lx, y, lz);
 
     // block finalization
     voxel& vox = chunk->voxels[(y * CHUNK_D + lz) * CHUNK_W + lx];
@@ -373,6 +375,13 @@ void Chunks::set(
     if (prevdef.rt.extended && !vox.state.segment) {
         eraseSegments(prevdef, vox.state, x, y, z);
     }
+    if (prevdef.dataStruct) {
+        if (auto found = chunk->blocksMetadata.find(index)) {
+            chunk->blocksMetadata.free(found);
+            chunk->flags.unsaved = true;
+            chunk->flags.blocksData = true;
+        }
+    }
 
     // block initialization
     const auto& newdef = indices->blocks.require(id);

src/voxels/ChunksStorage.cpp

@@ -59,8 +59,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
 
     auto chunk = std::make_shared<Chunk>(x, z);
     store(chunk);
-    auto data = regions.getChunk(chunk->x, chunk->z);
+    if (auto data = regions.getVoxels(chunk->x, chunk->z)) {
-    if (data) {
         chunk->decode(data.get());
 
         auto invs = regions.fetchInventories(chunk->x, chunk->z);
@@ -78,17 +77,17 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
         }
         verifyLoadedChunk(level->content->getIndices(), chunk.get());
     }
-
+    if (auto lights = regions.getLights(chunk->x, chunk->z)) {
-    auto lights = regions.getLights(chunk->x, chunk->z);
-    if (lights) {
         chunk->lightmap.set(lights.get());
         chunk->flags.loadedLights = true;
     }
+    chunk->blocksMetadata = regions.getBlocksData(chunk->x, chunk->z);
     return chunk;
 }
 
 // reduce nesting on next modification
 // 25.06.2024: not now
+// TODO: move to Chunks for performance improvement
 void ChunksStorage::getVoxels(VoxelsVolume* volume, bool backlight) const {
     const Content* content = level->content;
     auto indices = content->getIndices();

src/world/World.cpp

@@ -5,7 +5,7 @@
 #include <utility>
 
 #include "content/Content.hpp"
-#include "content/ContentLUT.hpp"
+#include "content/ContentReport.hpp"
 #include "debug/Logger.hpp"
 #include "files/WorldFiles.hpp"
 #include "items/Inventories.hpp"
@@ -156,12 +156,12 @@ std::unique_ptr<Level> World::load(
     return level;
 }
 
-std::shared_ptr<ContentLUT> World::checkIndices(
+std::shared_ptr<ContentReport> World::checkIndices(
     const std::shared_ptr<WorldFiles>& worldFiles, const Content* content
 ) {
     fs::path indicesFile = worldFiles->getIndicesFile();
     if (fs::is_regular_file(indicesFile)) {
-        return ContentLUT::create(worldFiles, indicesFile, content);
+        return ContentReport::create(worldFiles, indicesFile, content);
     }
     return nullptr;
 }

src/world/World.hpp

@@ -13,7 +13,7 @@
 class Content;
 class WorldFiles;
 class Level;
-class ContentLUT;
+class ContentReport;
 struct EngineSettings;
 
 namespace fs = std::filesystem;
@@ -80,11 +80,11 @@ public:
     /// @brief Write all unsaved level data to the world directory
     void write(Level* level);
 
-    /// @brief Check world indices and generate ContentLUT if convert required
+    /// @brief Check world indices and generate ContentReport if convert required
     /// @param directory world directory
     /// @param content current Content instance
-    /// @return ContentLUT if world convert required else nullptr
+    /// @return ContentReport if world convert required else nullptr
-    static std::shared_ptr<ContentLUT> checkIndices(
+    static std::shared_ptr<ContentReport> checkIndices(
         const std::shared_ptr<WorldFiles>& worldFiles, const Content* content
     );
 

test/coders/byte_utils.cpp

@@ -0,0 +1,20 @@
+#include <gtest/gtest.h>
+
+#include "coders/byte_utils.hpp"
+
+TEST(byte_utils, BytesBuildAndRead) {
+    ByteBuilder builder;
+    builder.putCStr("MAGIC.#");
+    builder.put(50);
+    builder.putInt16(1980);
+    builder.putInt32(123456789);
+    builder.putInt64(98765432123456789LL);
+    auto data = builder.build();
+
+    ByteReader reader(data.data(), data.size());
+    reader.checkMagic("MAGIC.#", 8);
+    EXPECT_EQ(reader.get(), 50);
+    EXPECT_EQ(reader.getInt16(), 1980);
+    EXPECT_EQ(reader.getInt32(), 123456789);
+    EXPECT_EQ(reader.getInt64(), 98765432123456789LL);
+}

test/coders/rle.cpp

@@ -3,46 +3,48 @@
 #include "typedefs.hpp"
 #include "coders/rle.hpp"
 
-TEST(RLE, EncodeDecode) {
+static void test_encode_decode(
-    const int initial_size = 50'000;
+    size_t(*encodefunc)(const ubyte*, size_t, ubyte*),
+    size_t(*decodefunc)(const ubyte*, size_t, ubyte*),
+    int dencity
+) {
+    const size_t initial_size = 50'000;
     uint8_t initial[initial_size];
     uint8_t next = rand();
-    for (int i = 0; i < initial_size; i++) {
+    for (size_t i = 0; i < initial_size; i++) {
         initial[i] = next;
-        if (rand() % 13 == 0) {
+        if (rand() % dencity == 0) {
             next = rand();
         }
     }
     uint8_t encoded[initial_size * 2];
-    auto encoded_size = rle::encode(initial, initial_size, encoded);
+    size_t encoded_size = encodefunc(initial, initial_size, encoded);
     uint8_t decoded[initial_size * 2];
-    auto decoded_size = rle::decode(encoded, encoded_size, decoded);
+    size_t decoded_size = decodefunc(encoded, encoded_size, decoded);
     
     EXPECT_EQ(decoded_size, initial_size);
 
-    for (int i = 0; i < decoded_size; i++) {
+    for (size_t i = 0; i < decoded_size; i++) {
         EXPECT_EQ(decoded[i], initial[i]);
     }
 }
 
+TEST(RLE, EncodeDecode) {
+    test_encode_decode(rle::encode, rle::decode, 13);
+    test_encode_decode(rle::encode, rle::decode, 90123);
+}
+
+TEST(RLE16, EncodeDecode) {
+    test_encode_decode(rle::encode16, rle::decode16, 13);
+    test_encode_decode(rle::encode16, rle::decode16, 90123);
+}
+
 TEST(ExtRLE, EncodeDecode) {
-    const int initial_size = 50'000;
+    test_encode_decode(extrle::encode, extrle::decode, 13);
-    uint8_t initial[initial_size];
+    test_encode_decode(extrle::encode, extrle::decode, 90123);
-    uint8_t next = rand();
+}
-    for (int i = 0; i < initial_size; i++) {
+
-        initial[i] = next;
+TEST(ExtRLE16, EncodeDecode) {
-        if (rand() % 13 == 0) {
+    test_encode_decode(extrle::encode16, extrle::decode16, 13);
-            next = rand();
+    test_encode_decode(extrle::encode16, extrle::decode16, 90123);
-        }
-    }
-    uint8_t encoded[initial_size * 2];
-    auto encoded_size = extrle::encode(initial, initial_size, encoded);
-    uint8_t decoded[initial_size * 2];
-    auto decoded_size = extrle::decode(encoded, encoded_size, decoded);
-    
-    EXPECT_EQ(decoded_size, initial_size);
-
-    for (int i = 0; i < decoded_size; i++) {
-        EXPECT_EQ(decoded[i], initial[i]);
-    }
 }

test/data/StructLayout.cpp

@@ -0,0 +1,133 @@
+#include "data/StructLayout.hpp"
+
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <climits>
+
+using namespace data;
+
+TEST(StructLayout, ReadWrite) {
+    ubyte buffer[16] {};
+
+    std::vector<Field> fields {
+        Field {FieldType::I8, "a", 1},
+        Field {FieldType::CHAR, "s", 4},
+        Field {FieldType::F32, "f", 1},
+    };
+    auto layout = StructLayout::create(fields);
+    EXPECT_EQ(layout.size(), 9);
+
+    layout.setInteger(buffer, 42, "a");
+    EXPECT_EQ(layout.getInteger(buffer, "a"), 42);
+
+    layout.setNumber(buffer, 3.141592f, "f");
+    EXPECT_FLOAT_EQ(layout.getNumber(buffer, "f"), 3.141592f);
+
+    layout.setAscii(buffer, "hello", "s");
+    EXPECT_EQ(layout.getChars(buffer, "s"), "hell");
+}
+
+TEST(StructLayout, Unicode) {
+    ubyte buffer[8] {};
+    std::vector<Field> fields {
+        Field {FieldType::CHAR, "text", 5},
+    };
+     auto layout = StructLayout::create(fields);
+     EXPECT_EQ(layout.size(), 5);
+
+     layout.setUnicode(buffer, u8"テキストデモ", "text");
+     EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"テ"));
+
+     layout.setUnicode(buffer, u8"пример", "text");
+     EXPECT_EQ(layout.getChars(buffer, "text"), std::string(u8"пр"));
+}
+
+TEST(StructLayout, ConvertReorder) {
+    ubyte src[16] {};
+    std::vector<Field> srcFields {
+        Field {FieldType::CHAR, "text", 5},
+        Field {FieldType::F64, "pi", 1},
+    };
+    auto srcLayout = StructLayout::create(srcFields);
+    srcLayout.setAscii(src, "truth", "text");
+    srcLayout.setNumber(src, 3.141592, "pi");
+
+    EXPECT_EQ(srcLayout.getChars(src, "text"), "truth");
+    EXPECT_DOUBLE_EQ(srcLayout.getNumber(src, "pi"), 3.141592);
+
+    ubyte dst[32] {};
+    std::vector<Field> dstFields {
+        Field {FieldType::I64, "arr", 2}, // an array of 2 i64
+        Field {FieldType::CHAR, "text", 5},
+        Field {FieldType::F64, "pi", 1},
+    };
+    auto dstLayout = StructLayout::create(dstFields);
+    EXPECT_EQ(srcLayout.checkCompatibility(dstLayout).size(), 0);
+    dstLayout.convert(srcLayout, src, dst, false);
+
+    EXPECT_EQ(dstLayout.getChars(dst, "text"), "truth");
+    EXPECT_DOUBLE_EQ(dstLayout.getNumber(dst, "pi"), 3.141592);
+}
+
+TEST(StructLayout, ConvertWithLoss) {
+    ubyte src[32] {};
+    std::vector<Field> srcFields {
+        Field {FieldType::CHAR, "text", 5},
+        Field {FieldType::I16, "someint", 1},
+        Field {FieldType::F64, "pi", 1},
+    };
+    auto srcLayout = StructLayout::create(srcFields);
+    srcLayout.setAscii(src, "truth", "text");
+    srcLayout.setInteger(src, 150, "someint");
+    srcLayout.setNumber(src, 3.141592, "pi");
+
+    EXPECT_EQ(srcLayout.getChars(src, "text"), "truth");
+    EXPECT_EQ(srcLayout.getInteger(src, "someint"), 150);
+    EXPECT_DOUBLE_EQ(srcLayout.getNumber(src, "pi"), 3.141592);
+
+    ubyte dst[8] {};
+    std::vector<Field> dstFields {
+        Field {FieldType::CHAR, "text", 3},
+        Field {FieldType::I8, "someint", 1, FieldConvertStrategy::CLAMP},
+    };
+    auto dstLayout = StructLayout::create(dstFields);
+    auto report = srcLayout.checkCompatibility(dstLayout);
+
+    // check report
+    std::sort(report.begin(), report.end(), [](const auto& a, const auto& b) {
+        return a.name < b.name;
+    });
+    EXPECT_EQ(report.size(), 3);
+
+    EXPECT_EQ(report[0].name, "pi");
+    EXPECT_EQ(report[0].type, FieldIncapatibilityType::MISSING);
+
+    EXPECT_EQ(report[1].name, "someint");
+    EXPECT_EQ(report[1].type, FieldIncapatibilityType::DATA_LOSS);
+
+    EXPECT_EQ(report[2].name, "text");
+    EXPECT_EQ(report[2].type, FieldIncapatibilityType::DATA_LOSS);
+
+    // convert with losses
+    dstLayout.convert(srcLayout, src, dst, true);
+
+    EXPECT_EQ(dstLayout.getChars(dst, "text"), "tru");
+    EXPECT_EQ(dstLayout.getInteger(dst, "someint"), INT8_MAX);
+}
+
+TEST(StructLayout, Serialization) {
+    std::vector<Field> fields {
+        Field (FieldType::CHAR, "text", 5),
+        Field (FieldType::I16, "someint", 1),
+        Field (FieldType::F64, "pi", 1),
+    };
+    auto layout1 = StructLayout::create(fields);
+    auto serialized = layout1.serialize();
+
+    std::cout << serialized << std::endl;
+
+    StructLayout layout2;
+    layout2.deserialize(serialized);
+
+    EXPECT_EQ(layout1, layout2);
+}

test/util/SmallHeap.cpp

@@ -0,0 +1,84 @@
+#include <gtest/gtest.h>
+
+#include "util/SmallHeap.hpp"
+
+using namespace util;
+
+TEST(SmallHeap, Allocation) {
+    auto index = 0;
+    auto size = 4;
+
+    SmallHeap<uint16_t, uint8_t> map;
+    auto ptr = map.allocate(index, size);
+    EXPECT_EQ(map.sizeOf(ptr), size);
+    EXPECT_EQ(ptr, map.find(index));
+}
+
+TEST(SmallHeap, MultipleAllocation) {
+    SmallHeap<uint16_t, uint8_t> map;
+    map.allocate(32, 4);
+    map.allocate(1, 5);
+    EXPECT_EQ(map.sizeOf(map.find(32)), 4);
+    EXPECT_EQ(map.sizeOf(map.find(1)), 5);
+}
+
+TEST(SmallHeap, Free) {
+    SmallHeap<uint16_t, uint8_t> map;
+    map.free(map.allocate(5, 4));
+    EXPECT_EQ(map.find(5), nullptr);
+}
+
+TEST(SmallHeap, ReAllocationSame) {
+    SmallHeap<uint16_t, uint8_t> map;
+    auto ptr1 = map.allocate(1, 2);
+    auto ptr2 = map.allocate(1, 2);
+    EXPECT_EQ(ptr1, ptr2);
+}
+
+TEST(SmallHeap, ReAllocationDifferent) {
+    SmallHeap<uint16_t, uint8_t> map;
+    map.allocate(1, 34);
+    map.allocate(1, 2);
+    EXPECT_EQ(map.sizeOf(map.find(1)), 2);
+}
+
+TEST(SmallHeap, RandomFill) {
+    SmallHeap<uint16_t, uint8_t> map;
+    int n = 3'000;
+    map.allocate(n, 123);
+    for (int i = 0; i < n; i++) {
+        int index = rand() % n;
+        int size = rand() % 254 + 1;
+        map.allocate(index, size);
+    }
+    EXPECT_EQ(map.sizeOf(map.find(n)), 123);
+}
+
+TEST(SmallHeap, EncodeDecode) {
+    SmallHeap<uint16_t, uint8_t> map;
+    int n = 3'000;
+    map.allocate(n, 123);
+    for (int i = 0; i < n; i++) {
+        int index = rand() % n;
+        int size = rand() % 254 + 1;
+        map.allocate(index, size);
+    }
+    auto bytes = map.serialize();
+
+    SmallHeap<uint16_t, uint8_t> out;
+    out.deserialize(bytes.data(), bytes.size());
+    EXPECT_EQ(map, out);
+}
+
+TEST(SmallHeap, Iterator) {
+    SmallHeap<uint16_t, uint8_t> map;
+    map.allocate(1, 10);
+    map.allocate(2, 20);
+    map.allocate(4, 14);
+    
+    int sum = 0;
+    for (const auto& it : map) {
+        sum += it.size();
+    }
+    EXPECT_EQ(sum, 44);
+}

test/voxels/Chunk.cpp

@@ -0,0 +1,25 @@
+#include <gtest/gtest.h>
+
+#include "voxels/Chunk.hpp"
+
+TEST(Chunk, EncodeDecode) {
+    Chunk chunk1(0, 0);
+    for (uint i = 0; i < CHUNK_VOL; i++) {
+        chunk1.voxels[i].id = rand();
+        chunk1.voxels[i].state.rotation = rand();
+        chunk1.voxels[i].state.segment = rand();
+        chunk1.voxels[i].state.userbits = rand();
+    }
+    auto bytes = chunk1.encode();
+
+    Chunk chunk2(0, 0);
+    chunk2.decode(bytes.get());
+
+    for (uint i = 0; i < CHUNK_VOL; i++) {
+        EXPECT_EQ(chunk1.voxels[i].id, chunk2.voxels[i].id);
+        EXPECT_EQ(
+            blockstate2int(chunk1.voxels[i].state), 
+            blockstate2int(chunk2.voxels[i].state)
+        );
+    }
+}