PulseStudio/VoxelEngine
PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

update WorldConverter (WIP)

Browse Source
This commit is contained in:
MihailRis 2024-09-02 23:24:59 +03:00
parent 3f826a88d3
commit 728795f0f3
12 changed files with 299 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/content/ContentReport.cpp
View File

@ -14,10 +14,12 @@
ContentReport::ContentReport(
const ContentIndices* indices,
size_t blocksCount,
size_t itemsCount
size_t itemsCount,
uint regionsVersion
)
: blocks(blocksCount, indices->blocks, BLOCK_VOID, ContentType::BLOCK),
items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM)
items(itemsCount, indices->items, ITEM_VOID, ContentType::ITEM),
regionsVersion(regionsVersion)
{}
template <class T>
@ -38,6 +40,8 @@ std::shared_ptr<ContentReport> ContentReport::create(
}
auto root = files::read_json(filename);
// TODO: remove default value 2 in 0.24
uint regionsVersion = root->get("region-version", 2U);
auto blocklist = root->list("blocks");
auto itemlist = root->list("items");
@ -45,7 +49,8 @@ std::shared_ptr<ContentReport> ContentReport::create(
size_t blocks_c = get_entries_count(indices->blocks, blocklist);
size_t items_c = get_entries_count(indices->items, itemlist);
auto report = std::make_shared<ContentReport>(indices, blocks_c, items_c);
auto report = std::make_shared<ContentReport>(
indices, blocks_c, items_c, regionsVersion);
report->blocks.setup(blocklist.get(), content->blocks);
report->items.setup(itemlist.get(), content->items);
report->buildIssues();

18
src/content/ContentReport.hpp
View File

@ -9,17 +9,22 @@
#include "data/dynamic.hpp"
#include "typedefs.hpp"
#include "Content.hpp"
#include "files/world_regions_fwd.hpp"
namespace fs = std::filesystem;
enum class ContentIssueType {
REORDER,
MISSING,
REGION_FORMAT_UPDATE,
};
struct ContentIssue {
ContentIssueType issueType;
ContentType contentType;
union {
ContentType contentType;
RegionLayerIndex regionLayer;
};
};
struct ContentEntry {
@ -29,12 +34,16 @@ struct ContentEntry {
class WorldFiles;
/// @brief Content unit lookup table
/// @tparam T index type
/// @tparam U unit class
template <typename T, class U>
class ContentUnitLUT {
std::vector<T> indices;
std::vector<std::string> names;
bool missingContent = false;
bool reorderContent = false;
/// @brief index that will be used to mark missing unit
T missingValue;
ContentType type;
public:
@ -110,13 +119,15 @@ class ContentReport {
public:
ContentUnitLUT<blockid_t, Block> blocks;
ContentUnitLUT<itemid_t, ItemDef> items;
uint regionsVersion;
std::vector<ContentIssue> issues;
ContentReport(
const ContentIndices* indices,
size_t blocks,
size_t items
size_t items,
uint regionsVersion
);
static std::shared_ptr<ContentReport> create(
@ -131,6 +142,9 @@ public:
inline bool hasMissingContent() const {
return blocks.hasMissingContent() || items.hasMissingContent();
}
inline bool isUpgradeRequired() const {
return regionsVersion < REGION_FORMAT_VERSION;
}
void buildIssues();
const std::vector<ContentIssue>& getIssues() const;

163
src/files/WorldConverter.cpp
View File

@ -12,43 +12,111 @@
#include "objects/Player.hpp"
#include "util/ThreadPool.hpp"
#include "voxels/Chunk.hpp"
#include "items/Inventory.hpp"
#include "WorldFiles.hpp"
namespace fs = std::filesystem;
static debug::Logger logger("world-converter");
class ConverterWorker : public util::Worker<convert_task, int> {
class ConverterWorker : public util::Worker<ConvertTask, int> {
std::shared_ptr<WorldConverter> converter;
public:
ConverterWorker(std::shared_ptr<WorldConverter> converter)
: converter(std::move(converter)) {
}
int operator()(const std::shared_ptr<convert_task>& task) override {
int operator()(const std::shared_ptr<ConvertTask>& task) override {
converter->convert(*task);
return 0;
}
};
void WorldConverter::addRegionsTasks(
RegionLayerIndex layerid,
ConvertTaskType taskType
) {
const auto& regions = wfile->getRegions();
auto regionsFolder = regions.getRegionsFolder(layerid);
if (!fs::is_directory(regionsFolder)) {
return;
}
for (const auto& file : fs::directory_iterator(regionsFolder)) {
int x, z;
std::string name = file.path().stem().string();
if (!WorldRegions::parseRegionFilename(name, x, z)) {
logger.error() << "could not parse region name " << name;
continue;
}
tasks.push(ConvertTask {taskType, file.path(), x, z});
}
}
void WorldConverter::createUpgradeTasks() {
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) {
continue;
}
if (issue.regionLayer == REGION_LAYER_VOXELS) {
addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS);
} else {
addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_SIMPLE);
}
}
}
void WorldConverter::createConvertTasks() {
auto handleReorder = [=](ContentType contentType) {
switch (contentType) {
case ContentType::BLOCK:
addRegionsTasks(
REGION_LAYER_VOXELS,
ConvertTaskType::VOXELS
);
break;
case ContentType::ITEM:
addRegionsTasks(
REGION_LAYER_INVENTORIES,
ConvertTaskType::INVENTORIES
);
break;
default:
break;
}
};
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
switch (issue.issueType) {
case ContentIssueType::REGION_FORMAT_UPDATE:
break;
case ContentIssueType::MISSING:
throw std::runtime_error("issue can't be resolved");
case ContentIssueType::REORDER:
handleReorder(issue.contentType);
break;
}
}
tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()});
}
WorldConverter::WorldConverter(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
std::shared_ptr<ContentReport> report
std::shared_ptr<ContentReport> reportPtr,
bool upgradeMode
)
: wfile(worldFiles),
report(std::move(report)),
content(content) {
fs::path regionsFolder =
wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS);
if (!fs::is_directory(regionsFolder)) {
logger.error() << "nothing to convert";
return;
}
tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()}
);
for (const auto& file : fs::directory_iterator(regionsFolder)) {
tasks.push(convert_task {convert_task_type::region, file.path()});
report(std::move(reportPtr)),
content(content),
upgradeMode(upgradeMode)
{
if (upgradeMode) {
createUpgradeTasks();
} else {
createConvertTasks();
}
}
@ -60,9 +128,11 @@ std::shared_ptr<Task> WorldConverter::startTask(
const Content* content,
const std::shared_ptr<ContentReport>& report,
const runnable& onDone,
bool upgradeMode,
bool multithreading
) {
auto converter = std::make_shared<WorldConverter>(worldFiles, content, report);
auto converter = std::make_shared<WorldConverter>(
worldFiles, content, report, upgradeMode);
if (!multithreading) {
converter->setOnComplete([=]() {
converter->write();
@ -70,15 +140,15 @@ std::shared_ptr<Task> WorldConverter::startTask(
});
return converter;
}
auto pool = std::make_shared<util::ThreadPool<convert_task, int>>(
auto pool = std::make_shared<util::ThreadPool<ConvertTask, int>>(
"converter-pool",
[=]() { return std::make_shared<ConverterWorker>(converter); },
[=](int&) {}
);
auto& converterTasks = converter->tasks;
while (!converterTasks.empty()) {
const convert_task& task = converterTasks.front();
auto ptr = std::make_shared<convert_task>(task);
const ConvertTask& task = converterTasks.front();
auto ptr = std::make_shared<ConvertTask>(task);
pool->enqueueJob(ptr);
converterTasks.pop();
}
@ -89,19 +159,27 @@ std::shared_ptr<Task> WorldConverter::startTask(
return pool;
}
void WorldConverter::convertRegion(const fs::path& file) const {
int x, z;
std::string name = file.stem().string();
if (!WorldRegions::parseRegionFilename(name, x, z)) {
logger.error() << "could not parse name " << name;
return;
}
logger.info() << "converting region " << name;
wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) {
if (report) {
Chunk::convert(data, report.get());
}
return true;
void WorldConverter::upgradeSimple(const fs::path& file, int x, int z) const {
throw std::runtime_error("unsupported region format");
}
void WorldConverter::upgradeVoxels(const fs::path& file, int x, int z) const {
throw std::runtime_error("unsupported region format");
}
void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const {
logger.info() << "converting voxels region " << x << "_" << z;
wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS, CHUNK_DATA_LEN,
[=](std::unique_ptr<ubyte[]> data, uint32_t*) {
Chunk::convert(data.get(), report.get());
return data;
});
}
void WorldConverter::convertInventories(const fs::path& file, int x, int z) const {
logger.info() << "converting inventories region " << x << "_" << z;
wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) {
inventory->convert(report.get());
});
}
@ -112,14 +190,23 @@ void WorldConverter::convertPlayer(const fs::path& file) const {
files::write_json(file, map.get());
}
void WorldConverter::convert(const convert_task& task) const {
void WorldConverter::convert(const ConvertTask& task) const {
if (!fs::is_regular_file(task.file)) return;
switch (task.type) {
case convert_task_type::region:
convertRegion(task.file);
case ConvertTaskType::UPGRADE_SIMPLE:
upgradeSimple(task.file, task.x, task.z);
break;
case convert_task_type::player:
case ConvertTaskType::UPGRADE_VOXELS:
upgradeVoxels(task.file, task.x, task.z);
break;
case ConvertTaskType::VOXELS:
convertVoxels(task.file, task.x, task.z);
break;
case ConvertTaskType::INVENTORIES:
convertInventories(task.file, task.x, task.z);
break;
case ConvertTaskType::PLAYER:
convertPlayer(task.file);
break;
}
@ -129,7 +216,7 @@ void WorldConverter::convertNext() {
if (tasks.empty()) {
throw std::runtime_error("no more regions to convert");
}
convert_task task = tasks.front();
ConvertTask task = tasks.front();
tasks.pop();
tasksDone++;
@ -157,7 +244,7 @@ bool WorldConverter::isActive() const {
void WorldConverter::write() {
logger.info() << "writing world";
wfile->write(nullptr, content);
wfile->write(nullptr, upgradeMode ? nullptr : content);
}
void WorldConverter::waitForEnd() {

43
src/files/WorldConverter.hpp
View File

@ -6,6 +6,7 @@
#include "delegates.hpp"
#include "interfaces/Task.hpp"
#include "files/world_regions_fwd.hpp"
#include "typedefs.hpp"
namespace fs = std::filesystem;
@ -14,32 +15,59 @@ class Content;
class ContentReport;
class WorldFiles;
enum class convert_task_type { region, player };
enum class ConvertTaskType {
/// @brief rewrite voxels region indices
VOXELS,
/// @brief rewrite inventories region indices
INVENTORIES,
/// @brief rewrite player
PLAYER,
/// @brief refresh region file version
UPGRADE_SIMPLE,
/// @brief rewrite voxels region file to new format
UPGRADE_VOXELS,
};
struct convert_task {
convert_task_type type;
struct ConvertTask {
ConvertTaskType type;
fs::path file;
/// @brief region coords
int x, z;
};
class WorldConverter : public Task {
std::shared_ptr<WorldFiles> wfile;
std::shared_ptr<ContentReport> const report;
const Content* const content;
std::queue<convert_task> tasks;
std::queue<ConvertTask> tasks;
runnable onComplete;
uint tasksDone = 0;
bool upgradeMode;
void upgradeSimple(const fs::path& file, int x, int z) const;
void upgradeVoxels(const fs::path& file, int x, int z) const;
void convertPlayer(const fs::path& file) const;
void convertRegion(const fs::path& file) const;
void convertVoxels(const fs::path& file, int x, int z) const;
void convertInventories(const fs::path& file, int x, int z) const;
void addRegionsTasks(
RegionLayerIndex layerid,
ConvertTaskType taskType
);
void createUpgradeTasks();
void createConvertTasks();
public:
WorldConverter(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
std::shared_ptr<ContentReport> report
std::shared_ptr<ContentReport> report,
bool upgradeMode
);
~WorldConverter();
void convert(const convert_task& task) const;
void convert(const ConvertTask& task) const;
void convertNext();
void setOnComplete(runnable callback);
void write();
@ -56,6 +84,7 @@ public:
const Content* content,
const std::shared_ptr<ContentReport>& report,
const runnable& onDone,
bool upgradeMode,
bool multithreading
);
};

10
src/files/WorldFiles.cpp
View File

@ -74,7 +74,9 @@ fs::path WorldFiles::getPacksFile() const {
return directory / fs::path("packs.list");
}
void WorldFiles::write(const World* world, const Content* content) {
void WorldFiles::write(
const World* world, const Content* content
) {
if (world) {
writeWorldInfo(world->getInfo());
if (!fs::exists(getPacksFile())) {
@ -84,8 +86,10 @@ void WorldFiles::write(const World* world, const Content* content) {
if (generatorTestMode) {
return;
}
writeIndices(content->getIndices());
regions.write();
if (content) {
writeIndices(content->getIndices());
}
regions.writeAll();
}
void WorldFiles::writePacks(const std::vector<ContentPack>& packs) {

84
src/files/WorldRegions.cpp
View File

@ -101,9 +101,8 @@ void WorldRegions::put(
}
static std::unique_ptr<ubyte[]> write_inventories(
Chunk* chunk, uint& datasize
const chunk_inventories_map& inventories, uint32_t& datasize
) {
auto& inventories = chunk->inventories;
ByteBuilder builder;
builder.putInt32(inventories.size());
for (auto& entry : inventories) {
@ -120,6 +119,22 @@ static std::unique_ptr<ubyte[]> write_inventories(
return data;
}
static chunk_inventories_map load_inventories(const ubyte* src, uint32_t size) {
chunk_inventories_map inventories;
ByteReader reader(src, size);
auto count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
inventories[index] = inv;
}
return inventories;
}
void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
assert(chunk != nullptr);
if (!chunk->flags.lighted) {
@ -150,7 +165,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
// Writing block inventories
if (!chunk->inventories.empty()) {
uint datasize;
auto data = write_inventories(chunk, datasize);
auto data = write_inventories(chunk->inventories, datasize);
put(chunk->x,
chunk->z,
REGION_LAYER_INVENTORIES,
@ -195,24 +210,25 @@ std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
}
chunk_inventories_map WorldRegions::fetchInventories(int x, int z) {
chunk_inventories_map meta;
uint32_t bytesSize;
auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize);
if (bytes == nullptr) {
return meta;
return {};
}
ByteReader reader(bytes, bytesSize);
auto count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
meta[index] = inv;
}
return meta;
return load_inventories(bytes, bytesSize);
}
void WorldRegions::processInventories(
int x, int z, const inventoryproc& func
) {
processRegion(x, z, REGION_LAYER_INVENTORIES, 0,
[=](std::unique_ptr<ubyte[]> data, uint32_t* size) {
auto inventories = load_inventories(data.get(), *size);
for (const auto& [_, inventory] : inventories) {
func(inventory.get());
}
return write_inventories(inventories, *size);
});
}
dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) {
@ -231,8 +247,10 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) {
return map;
}
void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
auto& layer = layers[REGION_LAYER_VOXELS];
void WorldRegions::processRegion(
int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func
) {
auto& layer = layers[layerid];
if (layer.getRegion(x, z)) {
throw std::runtime_error("not implemented for in-memory regions");
}
@ -250,25 +268,29 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
if (data == nullptr) {
continue;
}
data = compression::decompress(
data.get(), length, CHUNK_DATA_LEN, layer.compression
);
if (func(data.get())) {
put(gx,
gz,
REGION_LAYER_VOXELS,
std::move(data),
CHUNK_DATA_LEN);
uint32_t totalLength = dataLen;
if (layer.compression != compression::Method::NONE) {
if (dataLen == 0) {
throw std::invalid_argument("invalid data length");
}
data = compression::decompress(
data.get(), length, dataLen, layer.compression
);
} else {
totalLength = length;
}
if (auto writeData = func(std::move(data), &totalLength)) {
put(gx, gz, layerid, std::move(writeData), totalLength);
}
}
}
}
fs::path WorldRegions::getRegionsFolder(int layer) const {
return layers[layer].folder;
const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const {
return layers[layerid].folder;
}
void WorldRegions::write() {
void WorldRegions::writeAll() {
for (auto& layer : layers) {
fs::create_directories(layer.folder);
layer.writeAll();

41
src/files/WorldRegions.hpp
View File

@ -16,6 +16,7 @@
#include "maths/voxmaths.hpp"
#include "coders/compression.hpp"
#include "files.hpp"
#include "world_regions_fwd.hpp"
#define GLM_ENABLE_EXPERIMENTAL
#include <glm/gtx/hash.hpp>
@ -24,15 +25,6 @@ namespace fs = std::filesystem;
inline constexpr uint REGION_HEADER_SIZE = 10;
enum RegionLayerIndex : uint {
REGION_LAYER_VOXELS = 0,
REGION_LAYER_LIGHTS,
REGION_LAYER_INVENTORIES,
REGION_LAYER_ENTITIES,
REGION_LAYERS_COUNT
};
inline constexpr uint REGION_SIZE_BIT = 5;
inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT));
inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE));
@ -75,8 +67,10 @@ struct regfile {
};
using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using regionproc = std::function<bool(ubyte*)>;
using regionproc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
using inventoryproc = std::function<void(Inventory*)>;
/// @brief Region file pointer keeping inUse flag on until destroyed
class regfile_ptr {
regfile* file;
std::condition_variable* cv;
@ -209,6 +203,10 @@ public:
size_t size
);
/// @brief Get chunk voxels data
/// @param x chunk.x
/// @param z chunk.z
/// @return voxels data buffer or nullptr
std::unique_ptr<ubyte[]> getVoxels(int x, int z);
/// @brief Get cached lights for chunk at x,z
@ -217,13 +215,30 @@ public:
chunk_inventories_map fetchInventories(int x, int z);
/// @brief Load saved entities data for chunk
/// @param x chunk.x
/// @param z chunk.z
/// @return map with entities list as "data"
dynamic::Map_sptr fetchEntities(int x, int z);
void processRegionVoxels(int x, int z, const regionproc& func);
/// @brief Load, process and save processed region chunks data
/// @param x region X
/// @param z region Z
/// @param layerid regions layer index
/// @param func processing callback
void processRegion(
int x, int z, RegionLayerIndex layerid, uint32_t dataLen, const regionproc& func);
fs::path getRegionsFolder(int layer) const;
void processInventories(
int x, int z, const inventoryproc& func);
void write();
/// @brief Get regions directory by layer index
/// @param layerid layer index
/// @return directory path
const fs::path& getRegionsFolder(RegionLayerIndex layerid) const;
/// @brief Write all region layers
void writeAll();
/// @brief Extract X and Z from 'X_Z.bin' region file name.
/// @param name source region file name

12
src/files/world_regions_fwd.hpp Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#include "typedefs.hpp"
enum RegionLayerIndex : uint {
REGION_LAYER_VOXELS = 0,
REGION_LAYER_LIGHTS,
REGION_LAYER_INVENTORIES,
REGION_LAYER_ENTITIES,
REGION_LAYERS_COUNT
};

9
src/items/Inventory.cpp
View File

@ -81,6 +81,15 @@ std::unique_ptr<dynamic::Map> Inventory::serialize() const {
return map;
}
void Inventory::convert(const ContentReport* report) {
for (auto& slot : slots) {
itemid_t id = slot.getItemId();
itemid_t replacement = report->items.getId(id);
slot.set(ItemStack(replacement, slot.getCount()));
}
}
// TODO: remove
void Inventory::convert(dynamic::Map* data, const ContentReport* report) {
auto slotsarr = data->list("slots");
for (size_t i = 0; i < slotsarr->size(); i++) {

1
src/items/Inventory.hpp
View File

@ -42,6 +42,7 @@ public:
/* serializing inventory */
std::unique_ptr<dynamic::Map> serialize() const override;
void convert(const ContentReport* report);
static void convert(dynamic::Map* data, const ContentReport* report);
inline void setId(int64_t id) {

3
src/items/ItemStack.cpp
View File

@ -16,6 +16,9 @@ void ItemStack::set(const ItemStack& item) {
if (count == 0) {
this->item = 0;
}
if (this->item == 0) {
count = 0;
}
}
bool ItemStack::accepts(const ItemStack& other) const {

1
src/logic/EngineController.cpp
View File

@ -59,6 +59,7 @@ std::shared_ptr<Task> create_converter(
menu->setPage("main", false);
engine->getGUI()->postRunnable([=]() { postRunnable(); });
},
report->isUpgradeRequired(),
true
);
}