implement blocks data conversion

res/texts/en_US.txt

@@ -2,6 +2,7 @@
 menu.missing-content=Missing Content!
 world.convert-request=Content indices have changed! Convert world files?
 world.upgrade-request=World format is outdated! Convert world files?
+world.convert-with-loss=Convert world with data loss?
 pack.remove-confirm=Do you want to erase all pack(s) content from the world forever?
 error.pack-not-found=Could not to find pack
 error.dependency-not-found=Dependency pack is not found

res/texts/ru_RU.txt

@@ -47,6 +47,7 @@ world.generators.flat=Плоский
 world.Create World=Создать Мир
 world.convert-request=Есть изменения в индексах! Конвертировать мир?
 world.upgrade-request=Формат мира устарел! Конвертировать мир?
+world.convert-with-loss=Конвертировать мир с потерями?
 world.delete-confirm=Удалить мир безвозвратно?
 
 # Настройки

src/content/ContentReport.cpp

@@ -56,11 +56,40 @@ std::shared_ptr<ContentReport> ContentReport::create(
         indices, blocks_c, items_c, regionsVersion);
     report->blocks.setup(blocklist, content->blocks);
     report->items.setup(itemlist, content->items);
+
+    for (const auto& [name, map] : root["blocks-data"].asObject()) {
+        data::StructLayout layout;
+        layout.deserialize(map);
+        auto def = content->blocks.find(name);
+        if (def == nullptr) {
+            continue;
+        }
+        if (def->dataStruct == nullptr) {
+            ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
+            report->issues.push_back(issue);
+            report->dataLoss.push_back(name+": discard data");
+            continue;
+        }
+        auto incapatibility = layout.checkCompatibility(*def->dataStruct);
+        if (!incapatibility.empty()) {
+            ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
+            report->issues.push_back(issue);
+            for (const auto& error : incapatibility) {
+                report->dataLoss.push_back(
+                    "[" + name + "] field " + error.name + " - " +
+                    data::to_string(error.type)
+                );
+            }
+        }
+        report->blocksDataLayouts[name] = std::move(layout);
+    }
+
     report->buildIssues();
 
     if (report->isUpgradeRequired() || 
         report->hasContentReorder() || 
-        report->hasMissingContent()) {
+        report->hasMissingContent() ||
+        report->hasDataLoss()) {
         return report;
     } else {
         return nullptr;

src/content/ContentReport.hpp

@@ -4,11 +4,13 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <unordered_map>
 
 #include "constants.hpp"
 #include "data/dv.hpp"
 #include "typedefs.hpp"
 #include "Content.hpp"
+#include "data/StructLayout.hpp"
 #include "files/world_regions_fwd.hpp"
 
 namespace fs = std::filesystem;
@@ -17,6 +19,7 @@ enum class ContentIssueType {
     REORDER,
     MISSING,
     REGION_FORMAT_UPDATE,
+    BLOCK_DATA_LAYOUTS_UPDATE,
 };
 
 struct ContentIssue {
@@ -121,7 +124,9 @@ public:
     ContentUnitLUT<itemid_t, ItemDef> items;
     uint regionsVersion;
 
+    std::unordered_map<std::string, data::StructLayout> blocksDataLayouts;
     std::vector<ContentIssue> issues;
+    std::vector<std::string> dataLoss;
 
     ContentReport(
         const ContentIndices* indices, 
@@ -136,6 +141,10 @@ public:
         const Content* content
     );
 
+    inline const std::vector<std::string>& getDataLoss() const {
+        return dataLoss;
+    }
+
     inline bool hasContentReorder() const {
         return blocks.hasContentReorder() || items.hasContentReorder();
     }
@@ -145,6 +154,9 @@ public:
     inline bool isUpgradeRequired() const {
         return regionsVersion < REGION_FORMAT_VERSION;
     }
+    inline bool hasDataLoss() const {
+        return !dataLoss.empty();
+    }
     void buildIssues();
 
     const std::vector<ContentIssue>& getIssues() const;

src/data/StructLayout.hpp

@@ -30,6 +30,12 @@ namespace data {
         TYPE_ERROR,
         MISSING,
     };
+    inline const char* to_string(FieldIncapatibilityType type) {
+        const char* names[] = {
+            "none", "data_loss", "type_error", "missing"
+        };
+        return names[static_cast<int>(type)];
+    }
 
     struct FieldIncapatibility {
         std::string name;

src/files/WorldConverter.cpp

@@ -13,6 +13,7 @@
 #include "util/ThreadPool.hpp"
 #include "voxels/Chunk.hpp"
 #include "items/Inventory.hpp"
+#include "voxels/Block.hpp"
 #include "WorldFiles.hpp"
 
 namespace fs = std::filesystem;
@@ -85,6 +86,7 @@ void WorldConverter::createConvertTasks() {
     const auto& regions = wfile->getRegions();
     for (auto& issue : report->getIssues()) {
         switch (issue.issueType) {
+            case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
             case ContentIssueType::REGION_FORMAT_UPDATE:
                 break;
             case ContentIssueType::MISSING:
@@ -111,8 +113,24 @@ WorldConverter::WorldConverter(
 {
     if (upgradeMode) {
         createUpgradeTasks();
-    } else {
+    } else if (report->hasContentReorder()) {
         createConvertTasks();
+    } else {
+        // blocks data conversion requires correct block indices
+        // so it must be done AFTER voxels conversion
+        const auto& regions = wfile->getRegions();
+        for (auto& issue : report->getIssues()) {
+            switch (issue.issueType) {
+                case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
+                    addRegionsTasks(
+                        REGION_LAYER_BLOCKS_DATA,
+                        ConvertTaskType::CONVERT_BLOCKS_DATA
+                    );
+                    break;
+                default:
+                    break;
+            }
+        }
     }
 }
 
@@ -187,6 +205,34 @@ void WorldConverter::convertPlayer(const fs::path& file) const {
     files::write_json(file, map);
 }
 
+void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const {
+    logger.info() << "converting blocks data";
+    wfile->getRegions().processBlocksData(x, z, 
+    [=](BlocksMetadata& heap, std::unique_ptr<ubyte[]> voxelsData) {
+        Chunk chunk(0, 0);
+        chunk.decode(voxelsData.get());
+
+        const auto& indices = content->getIndices()->blocks;
+
+        BlocksMetadata newHeap;
+        for (const auto& entry : heap) {
+            size_t index = entry.index;
+            const auto& def = indices.require(chunk.voxels[index].id);
+            const auto& newStruct = *def.dataStruct;
+            const auto& found = report.blocksDataLayouts.find(def.name);
+            if (found == report.blocksDataLayouts.end()) {
+                logger.error() << "no previous fields layout found for block" 
+                    << def.name << " - discard";
+                continue; 
+            }
+            const auto& prevStruct = found->second;
+            uint8_t* dst = newHeap.allocate(index, newStruct.size());
+            newStruct.convert(prevStruct, entry.data(), dst, true);
+        }
+        heap = std::move(newHeap);
+    });
+}
+
 void WorldConverter::convert(const ConvertTask& task) const {
     if (!fs::is_regular_file(task.file)) return;
 
@@ -203,6 +249,9 @@ void WorldConverter::convert(const ConvertTask& task) const {
         case ConvertTaskType::PLAYER:
             convertPlayer(task.file);
             break;
+        case ConvertTaskType::CONVERT_BLOCKS_DATA:
+            convertBlocksData(task.x, task.z, *report);
+            break;
     }
 }
 

src/files/WorldConverter.hpp

@@ -24,6 +24,8 @@ enum class ConvertTaskType {
     PLAYER,
     /// @brief refresh region file version
     UPGRADE_REGION,
+    /// @brief convert blocks data to updated layouts
+    CONVERT_BLOCKS_DATA,
 };
 
 struct ConvertTask {
@@ -49,6 +51,7 @@ class WorldConverter : public Task {
     void convertPlayer(const fs::path& file) const;
     void convertVoxels(const fs::path& file, int x, int z) const;
     void convertInventories(const fs::path& file, int x, int z) const;
+    void convertBlocksData(int x, int z, const ContentReport& report) const;
 
     void addRegionsTasks(
         RegionLayerIndex layerid,

src/files/WorldFiles.cpp

@@ -21,6 +21,7 @@
 #include "objects/EntityDef.hpp"
 #include "objects/Player.hpp"
 #include "physics/Hitbox.hpp"
+#include "data/StructLayout.hpp"
 #include "settings.hpp"
 #include "typedefs.hpp"
 #include "util/data_io.hpp"
@@ -116,6 +117,15 @@ void WorldFiles::writeIndices(const ContentIndices* indices) {
     write_indices(indices->blocks, root.list("blocks"));
     write_indices(indices->items, root.list("items"));
     write_indices(indices->entities, root.list("entities"));
+
+    auto& structsMap = root.object("blocks-data");
+    for (const auto* def : indices->blocks.getIterable()) {
+        if (def->dataStruct == nullptr) {
+            continue;
+        }
+        structsMap[def->name] = def->dataStruct->serialize();
+    }
+    
     files::write_json(getIndicesFile(), root);
 }
 

src/files/WorldRegions.cpp

@@ -4,6 +4,7 @@
 #include <utility>
 #include <vector>
 
+#include "debug/Logger.hpp"
 #include "coders/byte_utils.hpp"
 #include "coders/rle.hpp"
 #include "coders/binary_json.hpp"
@@ -13,6 +14,8 @@
 
 #define REGION_FORMAT_MAGIC ".VOXREG"
 
+static debug::Logger logger("world-regions");
+
 WorldRegion::WorldRegion()
     : chunksData(
           std::make_unique<std::unique_ptr<ubyte[]>[]>(REGION_CHUNKS_COUNT)
@@ -95,15 +98,21 @@ void WorldRegions::put(
 ) {
     size_t size = srcSize;
     auto& layer = layers[layerid];
-    if (layer.compression != compression::Method::NONE) {
-        data = compression::compress(
-            data.get(), size, size, layer.compression);
-    }
     int regionX, regionZ, localX, localZ;
     calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
 
     WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
     region->setUnsaved(true);
+    
+    if (data == nullptr) {
+        region->put(localX, localZ, nullptr, 0, 0);
+        return;
+    }
+
+    if (layer.compression != compression::Method::NONE) {
+        data = compression::compress(
+            data.get(), size, size, layer.compression);
+    }
     region->put(localX, localZ, std::move(data), size, srcSize);
 }
 
@@ -251,9 +260,7 @@ BlocksMetadata WorldRegions::getBlocksData(int x, int z) {
     return heap;
 }
 
-void WorldRegions::processInventories(
+void WorldRegions::processInventories(int x, int z, const InventoryProc& func) {
-    int x, int z, const inventoryproc& func
-) {
     processRegion(x, z, REGION_LAYER_INVENTORIES,
     [=](std::unique_ptr<ubyte[]> data, uint32_t* size) {
         auto inventories = load_inventories(data.get(), *size);
@@ -264,6 +271,65 @@ void WorldRegions::processInventories(
     });
 }
 
+void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) {
+    auto& voxLayer = layers[REGION_LAYER_VOXELS];
+    auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA];
+    if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) {
+        throw std::runtime_error("not implemented for in-memory regions");
+    }
+    auto datRegfile = datLayer.getRegFile({x, z});
+    if (datRegfile == nullptr) {
+        throw std::runtime_error("could not open region file");
+    }
+    auto voxRegfile = voxLayer.getRegFile({x, z});
+    if (voxRegfile == nullptr) {
+        logger.warning() << "missing voxels region - discard blocks data for "
+            << x << "_" << z;
+        abort(); // TODO: delete region file
+    }
+    for (uint cz = 0; cz < REGION_SIZE; cz++) {
+        for (uint cx = 0; cx < REGION_SIZE; cx++) {
+            int gx = cx + x * REGION_SIZE;
+            int gz = cz + z * REGION_SIZE;
+
+            uint32_t datLength;
+            uint32_t datSrcSize;
+            auto datData = RegionsLayer::readChunkData(
+                gx, gz, datLength, datSrcSize, datRegfile.get()
+            );
+            if (datData == nullptr) {
+                continue;
+            }
+            uint32_t voxLength;
+            uint32_t voxSrcSize;
+            auto voxData = RegionsLayer::readChunkData(
+                gx, gz, voxLength, voxSrcSize, voxRegfile.get()
+            );
+            if (voxData == nullptr) {
+                logger.warning()
+                    << "missing voxels for chunk (" << gx << ", " << gz << ")";
+                put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0);
+                continue;
+            }
+            voxData = compression::decompress(
+                voxData.get(), voxLength, voxSrcSize, voxLayer.compression
+            );
+
+            BlocksMetadata blocksData;
+            blocksData.deserialize(datData.get(), datLength);
+            try {
+                func(blocksData, std::move(voxData));
+            } catch (const std::exception& err) {
+                logger.error() << "an error ocurred while processing blocks "
+                    "data in chunk (" << gx << ", " << gz << "): " << err.what();
+                blocksData = {};
+            }
+            auto bytes = blocksData.serialize();
+            put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size());
+        }
+    }
+}
+
 dv::value WorldRegions::fetchEntities(int x, int z) {
     if (generatorTestMode) {
         return nullptr;
@@ -282,7 +348,7 @@ dv::value WorldRegions::fetchEntities(int x, int z) {
 }
 
 void WorldRegions::processRegion(
-    int x, int z, RegionLayerIndex layerid, const regionproc& func
+    int x, int z, RegionLayerIndex layerid, const RegionProc& func
 ) {
     auto& layer = layers[layerid];
     if (layer.getRegion(x, z)) {

src/files/WorldRegions.hpp

@@ -64,9 +64,10 @@ struct regfile {
     std::unique_ptr<ubyte[]> read(int index, uint32_t& size, uint32_t& srcSize);
 };
 
-using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
+using RegionsMap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
-using regionproc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
+using RegionProc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
-using inventoryproc = std::function<void(Inventory*)>;
+using InventoryProc = std::function<void(Inventory*)>;
+using BlockDataProc = std::function<void(BlocksMetadata&, std::unique_ptr<ubyte[]>)>;
 
 /// @brief Region file pointer keeping inUse flag on until destroyed
 class regfile_ptr {
@@ -125,7 +126,7 @@ struct RegionsLayer {
     compression::Method compression = compression::Method::NONE;
 
     /// @brief In-memory regions data
-    regionsmap regions;
+    RegionsMap regions;
 
     /// @brief In-memory regions map mutex
     std::mutex mapMutex;
@@ -231,10 +232,11 @@ public:
     /// @param layerid regions layer index
     /// @param func processing callback
     void processRegion(
-        int x, int z, RegionLayerIndex layerid, const regionproc& func);
+        int x, int z, RegionLayerIndex layerid, const RegionProc& func);
 
-    void processInventories(
+    void processInventories(int x, int z, const InventoryProc& func);
-        int x, int z, const inventoryproc& func);
+
+    void processBlocksData(int x, int z, const BlockDataProc& func);
 
     /// @brief Get regions directory by layer index
     /// @param layerid layer index

src/graphics/ui/gui_util.cpp

@@ -3,6 +3,7 @@
 #include "elements/Label.hpp"
 #include "elements/Menu.hpp"
 #include "elements/Button.hpp"
+#include "elements/TextBox.hpp"
 #include "gui_xml.hpp"
 
 #include "logic/scripting/scripting.hpp"
@@ -77,3 +78,47 @@ void guiutil::confirm(
     menu->addPage("<confirm>", panel);
     menu->setPage("<confirm>");
 }
+
+void guiutil::confirmWithMemo(
+        gui::GUI* gui, 
+        const std::wstring& text, 
+        const std::wstring& memo,
+        const runnable& on_confirm,
+        std::wstring yestext, 
+        std::wstring notext) {
+
+    if (yestext.empty()) yestext = langs::get(L"Yes");
+    if (notext.empty()) notext = langs::get(L"No");
+
+    auto menu = gui->getMenu();
+    auto panel = std::make_shared<Panel>(glm::vec2(600, 500), glm::vec4(8.0f), 8.0f);
+    panel->setColor(glm::vec4(0.0f, 0.0f, 0.0f, 0.5f));
+    panel->add(std::make_shared<Label>(text));
+    
+    auto textbox = std::make_shared<TextBox>(L"");
+    textbox->setMultiline(true);
+    textbox->setTextWrapping(true);
+    textbox->setSize(glm::vec2(600, 300));
+    textbox->setText(memo);
+    textbox->setEditable(false);
+    panel->add(textbox);
+
+    auto subpanel = std::make_shared<Panel>(glm::vec2(600, 53));
+    subpanel->setColor(glm::vec4(0));
+
+    subpanel->add(std::make_shared<Button>(yestext, glm::vec4(8.f), [=](GUI*){
+        if (on_confirm)
+            on_confirm();
+        menu->back();
+    }));
+
+    subpanel->add(std::make_shared<Button>(notext, glm::vec4(8.f), [=](GUI*){
+        menu->back();
+    }));
+
+    panel->add(subpanel);
+
+    panel->refresh();
+    menu->addPage("<confirm>", panel);
+    menu->setPage("<confirm>");
+}

src/graphics/ui/gui_util.hpp

@@ -24,4 +24,12 @@ namespace guiutil {
         const runnable& on_confirm=nullptr,
         std::wstring yestext=L"", 
         std::wstring notext=L"");
+
+    void confirmWithMemo(
+        gui::GUI* gui, 
+        const std::wstring& text, 
+        const std::wstring& memo,
+        const runnable& on_confirm=nullptr,
+        std::wstring yestext=L"", 
+        std::wstring notext=L"");
 }

src/logic/EngineController.cpp

@@ -71,17 +71,37 @@ void show_convert_request(
     const std::shared_ptr<WorldFiles>& worldFiles,
     const runnable& postRunnable
 ) {
-    guiutil::confirm(
+    auto on_confirm = [=]() {
-        engine->getGUI(),
-        langs::get(report->isUpgradeRequired() ? 
-            L"world.upgrade-request" : L"world.convert-request"),
-        [=]() {
             auto converter =
                 create_converter(engine, worldFiles, content, report, postRunnable);
             menus::show_process_panel(
                 engine, converter, L"Converting world..."
             );
-        },
+        };
+
+    std::wstring message = L"world.convert-request";
+    if (report->isUpgradeRequired()) {
+        message = L"world.upgrade-request";
+    } else if (report->hasDataLoss()) {
+        message = L"world.convert-with-loss";
+        std::wstring text;
+        for (const auto& line : report->getDataLoss()) {
+            text += util::str2wstr_utf8(line) + L"\n";
+        }
+        guiutil::confirmWithMemo(
+            engine->getGUI(),
+            langs::get(message),
+            text,
+            on_confirm,
+            L"",
+            langs::get(L"Cancel")
+        );
+        return;
+    }
+    guiutil::confirm(
+        engine->getGUI(),
+        langs::get(message),
+        on_confirm,
         L"",
         langs::get(L"Cancel")
     );

src/util/SmallHeap.hpp

@@ -160,5 +160,57 @@ namespace util {
             buffer.resize(size - sizeof(Tindex));
             std::memcpy(buffer.data(), src + sizeof(Tindex), buffer.size());
         }
+
+        struct const_iterator {
+        private:
+            const std::vector<uint8_t>& buffer;
+        public:
+            Tindex index;
+            size_t offset;
+
+            const_iterator(
+                const std::vector<uint8_t>& buffer, Tindex index, size_t offset
+            ) : buffer(buffer), index(index), offset(offset) {}
+
+            Tsize size() const {
+                return read_int_le<Tsize>(buffer.data() + offset, -1);
+            }
+
+            bool operator!=(const const_iterator& o) const {
+                return o.offset != offset;
+            }
+
+            const_iterator& operator++() {
+                offset += size();
+                if (offset == buffer.size()) {
+                    return *this;
+                }
+                index = read_int_le<Tindex>(buffer.data() + offset);
+                offset += sizeof(Tindex) + sizeof(Tsize);
+                return *this;
+            }
+
+            const_iterator& operator*() {
+                return *this;
+            }
+
+            const uint8_t* data() const {
+                return buffer.data() + offset;
+            }
+        };
+
+        const_iterator begin() const {
+            if (buffer.empty()) {
+                return end();
+            }
+            return const_iterator (
+                buffer,
+                read_int_le<Tindex>(buffer.data()),
+                sizeof(Tindex) + sizeof(Tsize));
+        }
+
+        const_iterator end() const {
+            return const_iterator (buffer, 0, buffer.size());
+        }
     };
 }

src/voxels/ChunksStorage.cpp

@@ -87,6 +87,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
 
 // reduce nesting on next modification
 // 25.06.2024: not now
+// TODO: move to Chunks for performance improvement
 void ChunksStorage::getVoxels(VoxelsVolume* volume, bool backlight) const {
     const Content* content = level->content;
     auto indices = content->getIndices();

test/util/SmallHeap.cpp

@@ -69,3 +69,16 @@ TEST(SmallHeap, EncodeDecode) {
     out.deserialize(bytes.data(), bytes.size());
     EXPECT_EQ(map, out);
 }
+
+TEST(SmallHeap, Iterator) {
+    SmallHeap<uint16_t, uint8_t> map;
+    map.allocate(1, 10);
+    map.allocate(2, 20);
+    map.allocate(4, 14);
+    
+    int sum = 0;
+    for (const auto& it : map) {
+        sum += it.size();
+    }
+    EXPECT_EQ(sum, 44);
+}