diff --git a/src/content/ContentReport.cpp b/src/content/ContentReport.cpp index 54b4098d..1e524309 100644 --- a/src/content/ContentReport.cpp +++ b/src/content/ContentReport.cpp @@ -14,10 +14,12 @@ ContentReport::ContentReport( const ContentIndices* indices, size_t blocksCount, - size_t itemsCount + size_t itemsCount, + uint regionsVersion ) : blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK), - items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM) + items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM), + regionsVersion(regionsVersion) {} template @@ -38,6 +40,8 @@ std::shared_ptr ContentReport::create( } auto root = files::read_json(filename); + // TODO: remove default value 2 in 0.24 + uint regionsVersion = root->get("region-version", 2U); auto blocklist = root->list("blocks"); auto itemlist = root->list("items"); @@ -45,7 +49,8 @@ std::shared_ptr ContentReport::create( size_t blocks_c = get_entries_count(indices->blocks, blocklist); size_t items_c = get_entries_count(indices->items, itemlist); - auto report = std::make_shared(indices, blocks_c, items_c); + auto report = std::make_shared( + indices, blocks_c, items_c, regionsVersion); report->blocks.setup(blocklist.get(), content->blocks); report->items.setup(itemlist.get(), content->items); report->buildIssues(); diff --git a/src/content/ContentReport.hpp b/src/content/ContentReport.hpp index d4c1a056..416be01e 100644 --- a/src/content/ContentReport.hpp +++ b/src/content/ContentReport.hpp @@ -9,17 +9,22 @@ #include "data/dynamic.hpp" #include "typedefs.hpp" #include "Content.hpp" +#include "files/world_regions_fwd.hpp" namespace fs = std::filesystem; enum class ContentIssueType { REORDER, MISSING, + REGION_FORMAT_UPDATE, }; struct ContentIssue { ContentIssueType issueType; - ContentType contentType; + union { + ContentType contentType; + RegionLayerIndex regionLayer; + }; }; struct ContentEntry { @@ -29,12 +34,16 @@ struct ContentEntry { class WorldFiles; +/// @brief Content unit lookup table +/// @tparam T index type +/// @tparam U unit class template class ContentUnitLUT { std::vector indices; std::vector names; bool missingContent = false; bool reorderContent = false; + /// @brief index that will be used to mark missing unit T missingValue; ContentType type; public: @@ -110,13 +119,15 @@ class ContentReport { public: ContentUnitLUT blocks; ContentUnitLUT items; + uint regionsVersion; std::vector issues; ContentReport( const ContentIndices* indices, size_t blocks, - size_t items + size_t items, + uint regionsVersion ); static std::shared_ptr create( @@ -131,6 +142,9 @@ public: inline bool hasMissingContent() const { return blocks.hasMissingContent() || items.hasMissingContent(); } + inline bool isUpgradeRequired() const { + return regionsVersion < REGION_FORMAT_VERSION; + } void buildIssues(); const std::vector& getIssues() const; diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index c94708dd..a6b977bb 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -12,43 +12,111 @@ #include "objects/Player.hpp" #include "util/ThreadPool.hpp" #include "voxels/Chunk.hpp" +#include "items/Inventory.hpp" #include "WorldFiles.hpp" namespace fs = std::filesystem; static debug::Logger logger("world-converter"); -class ConverterWorker : public util::Worker { +class ConverterWorker : public util::Worker { std::shared_ptr converter; public: ConverterWorker(std::shared_ptr converter) : converter(std::move(converter)) { } - int operator()(const std::shared_ptr& task) override { + int operator()(const std::shared_ptr& task) override { converter->convert(*task); return 0; } }; +void WorldConverter::addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType +) { + const auto& regions = wfile->getRegions(); + auto regionsFolder = regions.getRegionsFolder(layerid); + if (!fs::is_directory(regionsFolder)) { + return; + } + for (const auto& file : fs::directory_iterator(regionsFolder)) { + int x, z; + std::string name = file.path().stem().string(); + if (!WorldRegions::parseRegionFilename(name, x, z)) { + logger.error() << "could not parse region name " << name; + continue; + } + tasks.push(ConvertTask {taskType, file.path(), x, z}); + } +} + +void WorldConverter::createUpgradeTasks() { + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) { + continue; + } + if (issue.regionLayer == REGION_LAYER_VOXELS) { + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS); + } else { + addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_SIMPLE); + } + } +} + +void WorldConverter::createConvertTasks() { + auto handleReorder = [=](ContentType contentType) { + switch (contentType) { + case ContentType::BLOCK: + addRegionsTasks( + REGION_LAYER_VOXELS, + ConvertTaskType::VOXELS + ); + break; + case ContentType::ITEM: + addRegionsTasks( + REGION_LAYER_INVENTORIES, + ConvertTaskType::INVENTORIES + ); + break; + default: + break; + } + }; + + const auto& regions = wfile->getRegions(); + for (auto& issue : report->getIssues()) { + switch (issue.issueType) { + case ContentIssueType::REGION_FORMAT_UPDATE: + break; + case ContentIssueType::MISSING: + throw std::runtime_error("issue can't be resolved"); + case ContentIssueType::REORDER: + handleReorder(issue.contentType); + break; + } + } + + tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()}); +} + WorldConverter::WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr report + std::shared_ptr reportPtr, + bool upgradeMode ) : wfile(worldFiles), - report(std::move(report)), - content(content) { - fs::path regionsFolder = - wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS); - if (!fs::is_directory(regionsFolder)) { - logger.error() << "nothing to convert"; - return; - } - tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()} - ); - for (const auto& file : fs::directory_iterator(regionsFolder)) { - tasks.push(convert_task {convert_task_type::region, file.path()}); + report(std::move(reportPtr)), + content(content), + upgradeMode(upgradeMode) +{ + if (upgradeMode) { + createUpgradeTasks(); + } else { + createConvertTasks(); } } @@ -60,9 +128,11 @@ std::shared_ptr WorldConverter::startTask( const Content* content, const std::shared_ptr& report, const runnable& onDone, + bool upgradeMode, bool multithreading ) { - auto converter = std::make_shared(worldFiles, content, report); + auto converter = std::make_shared( + worldFiles, content, report, upgradeMode); if (!multithreading) { converter->setOnComplete([=]() { converter->write(); @@ -70,15 +140,15 @@ std::shared_ptr WorldConverter::startTask( }); return converter; } - auto pool = std::make_shared>( + auto pool = std::make_shared>( "converter-pool", [=]() { return std::make_shared(converter); }, [=](int&) {} ); auto& converterTasks = converter->tasks; while (!converterTasks.empty()) { - const convert_task& task = converterTasks.front(); - auto ptr = std::make_shared(task); + const ConvertTask& task = converterTasks.front(); + auto ptr = std::make_shared(task); pool->enqueueJob(ptr); converterTasks.pop(); } @@ -89,19 +159,27 @@ std::shared_ptr WorldConverter::startTask( return pool; } -void WorldConverter::convertRegion(const fs::path& file) const { - int x, z; - std::string name = file.stem().string(); - if (!WorldRegions::parseRegionFilename(name, x, z)) { - logger.error() << "could not parse name " << name; - return; - } - logger.info() << "converting region " << name; - wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) { - if (report) { - Chunk::convert(data, report.get()); - } - return true; +void WorldConverter::upgradeSimple(const fs::path& file, int x, int z) const { + throw std::runtime_error("unsupported region format"); +} + +void WorldConverter::upgradeVoxels(const fs::path& file, int x, int z) const { + throw std::runtime_error("unsupported region format"); +} + +void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const { + logger.info() << "converting voxels region " << x << "_" << z; + wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, CHUNK_DATA_LEN, + [=](std::unique_ptr data, uint32_t*) { + Chunk::convert(data.get(), report.get()); + return data; + }); +} + +void WorldConverter::convertInventories(const fs::path& file, int x, int z) const { + logger.info() << "converting inventories region " << x << "_" << z; + wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) { + inventory->convert(report.get()); }); } @@ -112,14 +190,23 @@ void WorldConverter::convertPlayer(const fs::path& file) const { files::write_json(file, map.get()); } -void WorldConverter::convert(const convert_task& task) const { +void WorldConverter::convert(const ConvertTask& task) const { if (!fs::is_regular_file(task.file)) return; switch (task.type) { - case convert_task_type::region: - convertRegion(task.file); + case ConvertTaskType::UPGRADE_SIMPLE: + upgradeSimple(task.file, task.x, task.z); break; - case convert_task_type::player: + case ConvertTaskType::UPGRADE_VOXELS: + upgradeVoxels(task.file, task.x, task.z); + break; + case ConvertTaskType::VOXELS: + convertVoxels(task.file, task.x, task.z); + break; + case ConvertTaskType::INVENTORIES: + convertInventories(task.file, task.x, task.z); + break; + case ConvertTaskType::PLAYER: convertPlayer(task.file); break; } @@ -129,7 +216,7 @@ void WorldConverter::convertNext() { if (tasks.empty()) { throw std::runtime_error("no more regions to convert"); } - convert_task task = tasks.front(); + ConvertTask task = tasks.front(); tasks.pop(); tasksDone++; @@ -157,7 +244,7 @@ bool WorldConverter::isActive() const { void WorldConverter::write() { logger.info() << "writing world"; - wfile->write(nullptr, content); + wfile->write(nullptr, upgradeMode ? nullptr : content); } void WorldConverter::waitForEnd() { diff --git a/src/files/WorldConverter.hpp b/src/files/WorldConverter.hpp index 3b14942a..d8a3ce5f 100644 --- a/src/files/WorldConverter.hpp +++ b/src/files/WorldConverter.hpp @@ -6,6 +6,7 @@ #include "delegates.hpp" #include "interfaces/Task.hpp" +#include "files/world_regions_fwd.hpp" #include "typedefs.hpp" namespace fs = std::filesystem; @@ -14,32 +15,59 @@ class Content; class ContentReport; class WorldFiles; -enum class convert_task_type { region, player }; +enum class ConvertTaskType { + /// @brief rewrite voxels region indices + VOXELS, + /// @brief rewrite inventories region indices + INVENTORIES, + /// @brief rewrite player + PLAYER, + /// @brief refresh region file version + UPGRADE_SIMPLE, + /// @brief rewrite voxels region file to new format + UPGRADE_VOXELS, +}; -struct convert_task { - convert_task_type type; +struct ConvertTask { + ConvertTaskType type; fs::path file; + + /// @brief region coords + int x, z; }; class WorldConverter : public Task { std::shared_ptr wfile; std::shared_ptr const report; const Content* const content; - std::queue tasks; + std::queue tasks; runnable onComplete; uint tasksDone = 0; + bool upgradeMode; + void upgradeSimple(const fs::path& file, int x, int z) const; + void upgradeVoxels(const fs::path& file, int x, int z) const; void convertPlayer(const fs::path& file) const; - void convertRegion(const fs::path& file) const; + void convertVoxels(const fs::path& file, int x, int z) const; + void convertInventories(const fs::path& file, int x, int z) const; + + void addRegionsTasks( + RegionLayerIndex layerid, + ConvertTaskType taskType + ); + + void createUpgradeTasks(); + void createConvertTasks(); public: WorldConverter( const std::shared_ptr& worldFiles, const Content* content, - std::shared_ptr report + std::shared_ptr report, + bool upgradeMode ); ~WorldConverter(); - void convert(const convert_task& task) const; + void convert(const ConvertTask& task) const; void convertNext(); void setOnComplete(runnable callback); void write(); @@ -56,6 +84,7 @@ public: const Content* content, const std::shared_ptr& report, const runnable& onDone, + bool upgradeMode, bool multithreading ); }; diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 01b9cb77..54e96759 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -74,7 +74,9 @@ fs::path WorldFiles::getPacksFile() const { return directory / fs::path("packs.list"); } -void WorldFiles::write(const World* world, const Content* content) { +void WorldFiles::write( + const World* world, const Content* content +) { if (world) { writeWorldInfo(world->getInfo()); if (!fs::exists(getPacksFile())) { @@ -84,8 +86,10 @@ void WorldFiles::write(const World* world, const Content* content) { if (generatorTestMode) { return; } - writeIndices(content->getIndices()); - regions.write(); + if (content) { + writeIndices(content->getIndices()); + } + regions.writeAll(); } void WorldFiles::writePacks(const std::vector& packs) { diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index cd0955c1..f3e1fd88 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -101,9 +101,8 @@ void WorldRegions::put( } static std::unique_ptr write_inventories( - Chunk* chunk, uint& datasize + const chunk_inventories_map& inventories, uint32_t& datasize ) { - auto& inventories = chunk->inventories; ByteBuilder builder; builder.putInt32(inventories.size()); for (auto& entry : inventories) { @@ -120,6 +119,22 @@ static std::unique_ptr write_inventories( return data; } +static chunk_inventories_map load_inventories(const ubyte* src, uint32_t size) { + chunk_inventories_map inventories; + ByteReader reader(src, size); + auto count = reader.getInt32(); + for (int i = 0; i < count; i++) { + uint index = reader.getInt32(); + uint size = reader.getInt32(); + auto map = json::from_binary(reader.pointer(), size); + reader.skip(size); + auto inv = std::make_shared(0, 0); + inv->deserialize(map.get()); + inventories[index] = inv; + } + return inventories; +} + void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { assert(chunk != nullptr); if (!chunk->flags.lighted) { @@ -150,7 +165,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { // Writing block inventories if (!chunk->inventories.empty()) { uint datasize; - auto data = write_inventories(chunk, datasize); + auto data = write_inventories(chunk->inventories, datasize); put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES, @@ -195,24 +210,25 @@ std::unique_ptr WorldRegions::getLights(int x, int z) { } chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { - chunk_inventories_map meta; uint32_t bytesSize; auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize); if (bytes == nullptr) { - return meta; + return {}; } - ByteReader reader(bytes, bytesSize); - auto count = reader.getInt32(); - for (int i = 0; i < count; i++) { - uint index = reader.getInt32(); - uint size = reader.getInt32(); - auto map = json::from_binary(reader.pointer(), size); - reader.skip(size); - auto inv = std::make_shared(0, 0); - inv->deserialize(map.get()); - meta[index] = inv; - } - return meta; + return load_inventories(bytes, bytesSize); +} + +void WorldRegions::processInventories( + int x, int z, const inventoryproc& func +) { + processRegion(x, z, REGION_LAYER_INVENTORIES, 0, + [=](std::unique_ptr data, uint32_t* size) { + auto inventories = load_inventories(data.get(), *size); + for (const auto& [_, inventory] : inventories) { + func(inventory.get()); + } + return write_inventories(inventories, *size); + }); } dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { @@ -231,8 +247,10 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { return map; } -void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - auto& layer = layers[REGION_LAYER_VOXELS]; +void WorldRegions::processRegion( + int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func +) { + auto& layer = layers[layerid]; if (layer.getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } @@ -250,25 +268,29 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { if (data == nullptr) { continue; } - data = compression::decompress( - data.get(), length, CHUNK_DATA_LEN, layer.compression - ); - if (func(data.get())) { - put(gx, - gz, - REGION_LAYER_VOXELS, - std::move(data), - CHUNK_DATA_LEN); + uint32_t totalLength = dataLen; + if (layer.compression != compression::Method::NONE) { + if (dataLen == 0) { + throw std::invalid_argument("invalid data length"); + } + data = compression::decompress( + data.get(), length, dataLen, layer.compression + ); + } else { + totalLength = length; + } + if (auto writeData = func(std::move(data), &totalLength)) { + put(gx, gz, layerid, std::move(writeData), totalLength); } } } } -fs::path WorldRegions::getRegionsFolder(int layer) const { - return layers[layer].folder; +const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const { + return layers[layerid].folder; } -void WorldRegions::write() { +void WorldRegions::writeAll() { for (auto& layer : layers) { fs::create_directories(layer.folder); layer.writeAll(); diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index eb2e803a..e4060002 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -16,6 +16,7 @@ #include "maths/voxmaths.hpp" #include "coders/compression.hpp" #include "files.hpp" +#include "world_regions_fwd.hpp" #define GLM_ENABLE_EXPERIMENTAL #include @@ -24,15 +25,6 @@ namespace fs = std::filesystem; inline constexpr uint REGION_HEADER_SIZE = 10; -enum RegionLayerIndex : uint { - REGION_LAYER_VOXELS = 0, - REGION_LAYER_LIGHTS, - REGION_LAYER_INVENTORIES, - REGION_LAYER_ENTITIES, - - REGION_LAYERS_COUNT -}; - inline constexpr uint REGION_SIZE_BIT = 5; inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); @@ -75,8 +67,10 @@ struct regfile { }; using regionsmap = std::unordered_map>; -using regionproc = std::function; +using regionproc = std::function(std::unique_ptr,uint32_t*)>; +using inventoryproc = std::function; +/// @brief Region file pointer keeping inUse flag on until destroyed class regfile_ptr { regfile* file; std::condition_variable* cv; @@ -209,6 +203,10 @@ public: size_t size ); + /// @brief Get chunk voxels data + /// @param x chunk.x + /// @param z chunk.z + /// @return voxels data buffer or nullptr std::unique_ptr getVoxels(int x, int z); /// @brief Get cached lights for chunk at x,z @@ -217,13 +215,30 @@ public: chunk_inventories_map fetchInventories(int x, int z); + /// @brief Load saved entities data for chunk + /// @param x chunk.x + /// @param z chunk.z + /// @return map with entities list as "data" dynamic::Map_sptr fetchEntities(int x, int z); - void processRegionVoxels(int x, int z, const regionproc& func); + /// @brief Load, process and save processed region chunks data + /// @param x region X + /// @param z region Z + /// @param layerid regions layer index + /// @param func processing callback + void processRegion( + int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func); - fs::path getRegionsFolder(int layer) const; + void processInventories( + int x, int z, const inventoryproc& func); - void write(); + /// @brief Get regions directory by layer index + /// @param layerid layer index + /// @return directory path + const fs::path& getRegionsFolder(RegionLayerIndex layerid) const; + + /// @brief Write all region layers + void writeAll(); /// @brief Extract X and Z from 'X_Z.bin' region file name. /// @param name source region file name diff --git a/src/files/world_regions_fwd.hpp b/src/files/world_regions_fwd.hpp new file mode 100644 index 00000000..e0398641 --- /dev/null +++ b/src/files/world_regions_fwd.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "typedefs.hpp" + +enum RegionLayerIndex : uint { + REGION_LAYER_VOXELS = 0, + REGION_LAYER_LIGHTS, + REGION_LAYER_INVENTORIES, + REGION_LAYER_ENTITIES, + + REGION_LAYERS_COUNT +}; diff --git a/src/items/Inventory.cpp b/src/items/Inventory.cpp index c2bc16ed..309ac57f 100644 --- a/src/items/Inventory.cpp +++ b/src/items/Inventory.cpp @@ -81,6 +81,15 @@ std::unique_ptr Inventory::serialize() const { return map; } +void Inventory::convert(const ContentReport* report) { + for (auto& slot : slots) { + itemid_t id = slot.getItemId(); + itemid_t replacement = report->items.getId(id); + slot.set(ItemStack(replacement, slot.getCount())); + } +} + +// TODO: remove void Inventory::convert(dynamic::Map* data, const ContentReport* report) { auto slotsarr = data->list("slots"); for (size_t i = 0; i < slotsarr->size(); i++) { diff --git a/src/items/Inventory.hpp b/src/items/Inventory.hpp index 260ec08c..0f80fb26 100644 --- a/src/items/Inventory.hpp +++ b/src/items/Inventory.hpp @@ -42,6 +42,7 @@ public: /* serializing inventory */ std::unique_ptr serialize() const override; + void convert(const ContentReport* report); static void convert(dynamic::Map* data, const ContentReport* report); inline void setId(int64_t id) { diff --git a/src/items/ItemStack.cpp b/src/items/ItemStack.cpp index 932f4417..99cb0a0e 100644 --- a/src/items/ItemStack.cpp +++ b/src/items/ItemStack.cpp @@ -16,6 +16,9 @@ void ItemStack::set(const ItemStack& item) { if (count == 0) { this->item = 0; } + if (this->item == 0) { + count = 0; + } } bool ItemStack::accepts(const ItemStack& other) const { diff --git a/src/logic/EngineController.cpp b/src/logic/EngineController.cpp index 91e77220..8cb53f28 100644 --- a/src/logic/EngineController.cpp +++ b/src/logic/EngineController.cpp @@ -59,6 +59,7 @@ std::shared_ptr create_converter( menu->setPage("main", false); engine->getGUI()->postRunnable([=]() { postRunnable(); }); }, + report->isUpgradeRequired(), true ); }