PulseStudio/VoxelEngine
PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

implement blocks data conversion

Browse Source
This commit is contained in:
MihailRis 2024-10-01 19:08:45 +03:00
parent 28d746f371
commit bc05716772
16 changed files with 341 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
res/texts/en_US.txt
View File

@ -2,6 +2,7 @@
menu.missing-content=Missing Content!
world.convert-request=Content indices have changed! Convert world files?
world.upgrade-request=World format is outdated! Convert world files?
world.convert-with-loss=Convert world with data loss?
pack.remove-confirm=Do you want to erase all pack(s) content from the world forever?
error.pack-not-found=Could not to find pack
error.dependency-not-found=Dependency pack is not found

1
res/texts/ru_RU.txt
View File

@ -47,6 +47,7 @@ world.generators.flat=Плоский
world.Create World=Создать Мир
world.convert-request=Есть изменения в индексах! Конвертировать мир?
world.upgrade-request=Формат мира устарел! Конвертировать мир?
world.convert-with-loss=Конвертировать мир с потерями?
world.delete-confirm=Удалить мир безвозвратно?
# Настройки

31
src/content/ContentReport.cpp
View File

@ -56,11 +56,40 @@ std::shared_ptr<ContentReport> ContentReport::create(
indices, blocks_c, items_c, regionsVersion);
report->blocks.setup(blocklist, content->blocks);
report->items.setup(itemlist, content->items);
for (const auto& [name, map] : root["blocks-data"].asObject()) {
data::StructLayout layout;
layout.deserialize(map);
auto def = content->blocks.find(name);
if (def == nullptr) {
continue;
}
if (def->dataStruct == nullptr) {
ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
report->issues.push_back(issue);
report->dataLoss.push_back(name+": discard data");
continue;
}
auto incapatibility = layout.checkCompatibility(*def->dataStruct);
if (!incapatibility.empty()) {
ContentIssue issue {ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE};
report->issues.push_back(issue);
for (const auto& error : incapatibility) {
report->dataLoss.push_back(
"[" + name + "] field " + error.name + " - " +
data::to_string(error.type)
);
}
}
report->blocksDataLayouts[name] = std::move(layout);
}
report->buildIssues();
if (report->isUpgradeRequired() ||
report->hasContentReorder() ||
report->hasMissingContent()) {
report->hasMissingContent() ||
report->hasDataLoss()) {
return report;
} else {
return nullptr;

12
src/content/ContentReport.hpp
View File

@ -4,11 +4,13 @@
#include <string>
#include <utility>
#include <vector>
#include <unordered_map>
#include "constants.hpp"
#include "data/dv.hpp"
#include "typedefs.hpp"
#include "Content.hpp"
#include "data/StructLayout.hpp"
#include "files/world_regions_fwd.hpp"
namespace fs = std::filesystem;
@ -17,6 +19,7 @@ enum class ContentIssueType {
REORDER,
MISSING,
REGION_FORMAT_UPDATE,
BLOCK_DATA_LAYOUTS_UPDATE,
};
struct ContentIssue {
@ -121,7 +124,9 @@ public:
ContentUnitLUT<itemid_t, ItemDef> items;
uint regionsVersion;
std::unordered_map<std::string, data::StructLayout> blocksDataLayouts;
std::vector<ContentIssue> issues;
std::vector<std::string> dataLoss;
ContentReport(
const ContentIndices* indices,
@ -136,6 +141,10 @@ public:
const Content* content
);
inline const std::vector<std::string>& getDataLoss() const {
return dataLoss;
}
inline bool hasContentReorder() const {
return blocks.hasContentReorder() || items.hasContentReorder();
}
@ -145,6 +154,9 @@ public:
inline bool isUpgradeRequired() const {
return regionsVersion < REGION_FORMAT_VERSION;
}
inline bool hasDataLoss() const {
return !dataLoss.empty();
}
void buildIssues();
const std::vector<ContentIssue>& getIssues() const;

6
src/data/StructLayout.hpp
View File

@ -30,6 +30,12 @@ namespace data {
TYPE_ERROR,
MISSING,
};
inline const char* to_string(FieldIncapatibilityType type) {
const char* names[] = {
"none", "data_loss", "type_error", "missing"
};
return names[static_cast<int>(type)];
}
struct FieldIncapatibility {
std::string name;

51
src/files/WorldConverter.cpp
View File

@ -13,6 +13,7 @@
#include "util/ThreadPool.hpp"
#include "voxels/Chunk.hpp"
#include "items/Inventory.hpp"
#include "voxels/Block.hpp"
#include "WorldFiles.hpp"
namespace fs = std::filesystem;
@ -85,6 +86,7 @@ void WorldConverter::createConvertTasks() {
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
switch (issue.issueType) {
case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
case ContentIssueType::REGION_FORMAT_UPDATE:
break;
case ContentIssueType::MISSING:
@ -111,8 +113,24 @@ WorldConverter::WorldConverter(
{
if (upgradeMode) {
createUpgradeTasks();
} else {
} else if (report->hasContentReorder()) {
createConvertTasks();
} else {
// blocks data conversion requires correct block indices
// so it must be done AFTER voxels conversion
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
switch (issue.issueType) {
case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
addRegionsTasks(
REGION_LAYER_BLOCKS_DATA,
ConvertTaskType::CONVERT_BLOCKS_DATA
);
break;
default:
break;
}
}
}
}
@ -187,6 +205,34 @@ void WorldConverter::convertPlayer(const fs::path& file) const {
files::write_json(file, map);
}
void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const {
logger.info() << "converting blocks data";
wfile->getRegions().processBlocksData(x, z,
[=](BlocksMetadata& heap, std::unique_ptr<ubyte[]> voxelsData) {
Chunk chunk(0, 0);
chunk.decode(voxelsData.get());
const auto& indices = content->getIndices()->blocks;
BlocksMetadata newHeap;
for (const auto& entry : heap) {
size_t index = entry.index;
const auto& def = indices.require(chunk.voxels[index].id);
const auto& newStruct = *def.dataStruct;
const auto& found = report.blocksDataLayouts.find(def.name);
if (found == report.blocksDataLayouts.end()) {
logger.error() << "no previous fields layout found for block"
<< def.name << " - discard";
continue;
}
const auto& prevStruct = found->second;
uint8_t* dst = newHeap.allocate(index, newStruct.size());
newStruct.convert(prevStruct, entry.data(), dst, true);
}
heap = std::move(newHeap);
});
}
void WorldConverter::convert(const ConvertTask& task) const {
if (!fs::is_regular_file(task.file)) return;
@ -203,6 +249,9 @@ void WorldConverter::convert(const ConvertTask& task) const {
case ConvertTaskType::PLAYER:
convertPlayer(task.file);
break;
case ConvertTaskType::CONVERT_BLOCKS_DATA:
convertBlocksData(task.x, task.z, *report);
break;
}
}

3
src/files/WorldConverter.hpp
View File

@ -24,6 +24,8 @@ enum class ConvertTaskType {
PLAYER,
/// @brief refresh region file version
UPGRADE_REGION,
/// @brief convert blocks data to updated layouts
CONVERT_BLOCKS_DATA,
};
struct ConvertTask {
@ -49,6 +51,7 @@ class WorldConverter : public Task {
void convertPlayer(const fs::path& file) const;
void convertVoxels(const fs::path& file, int x, int z) const;
void convertInventories(const fs::path& file, int x, int z) const;
void convertBlocksData(int x, int z, const ContentReport& report) const;
void addRegionsTasks(
RegionLayerIndex layerid,

10
src/files/WorldFiles.cpp
View File

@ -21,6 +21,7 @@
#include "objects/EntityDef.hpp"
#include "objects/Player.hpp"
#include "physics/Hitbox.hpp"
#include "data/StructLayout.hpp"
#include "settings.hpp"
#include "typedefs.hpp"
#include "util/data_io.hpp"
@ -116,6 +117,15 @@ void WorldFiles::writeIndices(const ContentIndices* indices) {
write_indices(indices->blocks, root.list("blocks"));
write_indices(indices->items, root.list("items"));
write_indices(indices->entities, root.list("entities"));
auto& structsMap = root.object("blocks-data");
for (const auto* def : indices->blocks.getIterable()) {
if (def->dataStruct == nullptr) {
continue;
}
structsMap[def->name] = def->dataStruct->serialize();
}
files::write_json(getIndicesFile(), root);
}

82
src/files/WorldRegions.cpp
View File

@ -4,6 +4,7 @@
#include <utility>
#include <vector>
#include "debug/Logger.hpp"
#include "coders/byte_utils.hpp"
#include "coders/rle.hpp"
#include "coders/binary_json.hpp"
@ -13,6 +14,8 @@
#define REGION_FORMAT_MAGIC ".VOXREG"
static debug::Logger logger("world-regions");
WorldRegion::WorldRegion()
: chunksData(
std::make_unique<std::unique_ptr<ubyte[]>[]>(REGION_CHUNKS_COUNT)
@ -95,15 +98,21 @@ void WorldRegions::put(
) {
size_t size = srcSize;
auto& layer = layers[layerid];
if (layer.compression != compression::Method::NONE) {
data = compression::compress(
data.get(), size, size, layer.compression);
}
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
region->setUnsaved(true);
if (data == nullptr) {
region->put(localX, localZ, nullptr, 0, 0);
return;
}
if (layer.compression != compression::Method::NONE) {
data = compression::compress(
data.get(), size, size, layer.compression);
}
region->put(localX, localZ, std::move(data), size, srcSize);
}
@ -251,9 +260,7 @@ BlocksMetadata WorldRegions::getBlocksData(int x, int z) {
return heap;
}
void WorldRegions::processInventories(
int x, int z, const inventoryproc& func
) {
void WorldRegions::processInventories(int x, int z, const InventoryProc& func) {
processRegion(x, z, REGION_LAYER_INVENTORIES,
[=](std::unique_ptr<ubyte[]> data, uint32_t* size) {
auto inventories = load_inventories(data.get(), *size);
@ -264,6 +271,65 @@ void WorldRegions::processInventories(
});
}
void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) {
auto& voxLayer = layers[REGION_LAYER_VOXELS];
auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA];
if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto datRegfile = datLayer.getRegFile({x, z});
if (datRegfile == nullptr) {
throw std::runtime_error("could not open region file");
}
auto voxRegfile = voxLayer.getRegFile({x, z});
if (voxRegfile == nullptr) {
logger.warning() << "missing voxels region - discard blocks data for "
<< x << "_" << z;
abort(); // TODO: delete region file
}
for (uint cz = 0; cz < REGION_SIZE; cz++) {
for (uint cx = 0; cx < REGION_SIZE; cx++) {
int gx = cx + x * REGION_SIZE;
int gz = cz + z * REGION_SIZE;
uint32_t datLength;
uint32_t datSrcSize;
auto datData = RegionsLayer::readChunkData(
gx, gz, datLength, datSrcSize, datRegfile.get()
);
if (datData == nullptr) {
continue;
}
uint32_t voxLength;
uint32_t voxSrcSize;
auto voxData = RegionsLayer::readChunkData(
gx, gz, voxLength, voxSrcSize, voxRegfile.get()
);
if (voxData == nullptr) {
logger.warning()
<< "missing voxels for chunk (" << gx << ", " << gz << ")";
put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0);
continue;
}
voxData = compression::decompress(
voxData.get(), voxLength, voxSrcSize, voxLayer.compression
);
BlocksMetadata blocksData;
blocksData.deserialize(datData.get(), datLength);
try {
func(blocksData, std::move(voxData));
} catch (const std::exception& err) {
logger.error() << "an error ocurred while processing blocks "
"data in chunk (" << gx << ", " << gz << "): " << err.what();
blocksData = {};
}
auto bytes = blocksData.serialize();
put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size());
}
}
}
dv::value WorldRegions::fetchEntities(int x, int z) {
if (generatorTestMode) {
return nullptr;
@ -282,7 +348,7 @@ dv::value WorldRegions::fetchEntities(int x, int z) {
}
void WorldRegions::processRegion(
int x, int z, RegionLayerIndex layerid, const regionproc& func
int x, int z, RegionLayerIndex layerid, const RegionProc& func
) {
auto& layer = layers[layerid];
if (layer.getRegion(x, z)) {

16
src/files/WorldRegions.hpp
View File

@ -64,9 +64,10 @@ struct regfile {
std::unique_ptr<ubyte[]> read(int index, uint32_t& size, uint32_t& srcSize);
};
using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using regionproc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
using inventoryproc = std::function<void(Inventory*)>;
using RegionsMap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using RegionProc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
using InventoryProc = std::function<void(Inventory*)>;
using BlockDataProc = std::function<void(BlocksMetadata&, std::unique_ptr<ubyte[]>)>;
/// @brief Region file pointer keeping inUse flag on until destroyed
class regfile_ptr {
@ -125,7 +126,7 @@ struct RegionsLayer {
compression::Method compression = compression::Method::NONE;
/// @brief In-memory regions data
regionsmap regions;
RegionsMap regions;
/// @brief In-memory regions map mutex
std::mutex mapMutex;
@ -231,10 +232,11 @@ public:
/// @param layerid regions layer index
/// @param func processing callback
void processRegion(
int x, int z, RegionLayerIndex layerid, const regionproc& func);
int x, int z, RegionLayerIndex layerid, const RegionProc& func);
void processInventories(
int x, int z, const inventoryproc& func);
void processInventories(int x, int z, const InventoryProc& func);
void processBlocksData(int x, int z, const BlockDataProc& func);
/// @brief Get regions directory by layer index
/// @param layerid layer index

45
src/graphics/ui/gui_util.cpp
View File

@ -3,6 +3,7 @@
#include "elements/Label.hpp"
#include "elements/Menu.hpp"
#include "elements/Button.hpp"
#include "elements/TextBox.hpp"
#include "gui_xml.hpp"
#include "logic/scripting/scripting.hpp"
@ -77,3 +78,47 @@ void guiutil::confirm(
menu->addPage("<confirm>", panel);
menu->setPage("<confirm>");
}
void guiutil::confirmWithMemo(
gui::GUI* gui,
const std::wstring& text,
const std::wstring& memo,
const runnable& on_confirm,
std::wstring yestext,
std::wstring notext) {
if (yestext.empty()) yestext = langs::get(L"Yes");
if (notext.empty()) notext = langs::get(L"No");
auto menu = gui->getMenu();
auto panel = std::make_shared<Panel>(glm::vec2(600, 500), glm::vec4(8.0f), 8.0f);
panel->setColor(glm::vec4(0.0f, 0.0f, 0.0f, 0.5f));
panel->add(std::make_shared<Label>(text));
auto textbox = std::make_shared<TextBox>(L"");
textbox->setMultiline(true);
textbox->setTextWrapping(true);
textbox->setSize(glm::vec2(600, 300));
textbox->setText(memo);
textbox->setEditable(false);
panel->add(textbox);
auto subpanel = std::make_shared<Panel>(glm::vec2(600, 53));
subpanel->setColor(glm::vec4(0));
subpanel->add(std::make_shared<Button>(yestext, glm::vec4(8.f), [=](GUI*){
if (on_confirm)
on_confirm();
menu->back();
}));
subpanel->add(std::make_shared<Button>(notext, glm::vec4(8.f), [=](GUI*){
menu->back();
}));
panel->add(subpanel);
panel->refresh();
menu->addPage("<confirm>", panel);
menu->setPage("<confirm>");
}

8
src/graphics/ui/gui_util.hpp
View File

@ -24,4 +24,12 @@ namespace guiutil {
const runnable& on_confirm=nullptr,
std::wstring yestext=L"",
std::wstring notext=L"");
void confirmWithMemo(
gui::GUI* gui,
const std::wstring& text,
const std::wstring& memo,
const runnable& on_confirm=nullptr,
std::wstring yestext=L"",
std::wstring notext=L"");
}

32
src/logic/EngineController.cpp
View File

@ -71,17 +71,37 @@ void show_convert_request(
const std::shared_ptr<WorldFiles>& worldFiles,
const runnable& postRunnable
) {
guiutil::confirm(
engine->getGUI(),
langs::get(report->isUpgradeRequired() ?
L"world.upgrade-request" : L"world.convert-request"),
[=]() {
auto on_confirm = [=]() {
auto converter =
create_converter(engine, worldFiles, content, report, postRunnable);
menus::show_process_panel(
engine, converter, L"Converting world..."
);
},
};
std::wstring message = L"world.convert-request";
if (report->isUpgradeRequired()) {
message = L"world.upgrade-request";
} else if (report->hasDataLoss()) {
message = L"world.convert-with-loss";
std::wstring text;
for (const auto& line : report->getDataLoss()) {
text += util::str2wstr_utf8(line) + L"\n";
}
guiutil::confirmWithMemo(
engine->getGUI(),
langs::get(message),
text,
on_confirm,
L"",
langs::get(L"Cancel")
);
return;
}
guiutil::confirm(
engine->getGUI(),
langs::get(message),
on_confirm,
L"",
langs::get(L"Cancel")
);

52
src/util/SmallHeap.hpp
View File

@ -160,5 +160,57 @@ namespace util {
buffer.resize(size - sizeof(Tindex));
std::memcpy(buffer.data(), src + sizeof(Tindex), buffer.size());
}
struct const_iterator {
private:
const std::vector<uint8_t>& buffer;
public:
Tindex index;
size_t offset;
const_iterator(
const std::vector<uint8_t>& buffer, Tindex index, size_t offset
) : buffer(buffer), index(index), offset(offset) {}
Tsize size() const {
return read_int_le<Tsize>(buffer.data() + offset, -1);
}
bool operator!=(const const_iterator& o) const {
return o.offset != offset;
}
const_iterator& operator++() {
offset += size();
if (offset == buffer.size()) {
return *this;
}
index = read_int_le<Tindex>(buffer.data() + offset);
offset += sizeof(Tindex) + sizeof(Tsize);
return *this;
}
const_iterator& operator*() {
return *this;
}
const uint8_t* data() const {
return buffer.data() + offset;
}
};
const_iterator begin() const {
if (buffer.empty()) {
return end();
}
return const_iterator (
buffer,
read_int_le<Tindex>(buffer.data()),
sizeof(Tindex) + sizeof(Tsize));
}
const_iterator end() const {
return const_iterator (buffer, 0, buffer.size());
}
};
}

1
src/voxels/ChunksStorage.cpp
View File

@ -87,6 +87,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
// reduce nesting on next modification
// 25.06.2024: not now
// TODO: move to Chunks for performance improvement
void ChunksStorage::getVoxels(VoxelsVolume* volume, bool backlight) const {
const Content* content = level->content;
auto indices = content->getIndices();

13
test/util/SmallHeap.cpp
View File

@ -69,3 +69,16 @@ TEST(SmallHeap, EncodeDecode) {
out.deserialize(bytes.data(), bytes.size());
EXPECT_EQ(map, out);
}
TEST(SmallHeap, Iterator) {
SmallHeap<uint16_t, uint8_t> map;
map.allocate(1, 10);
map.allocate(2, 20);
map.allocate(4, 14);
int sum = 0;
for (const auto& it : map) {
sum += it.size();
}
EXPECT_EQ(sum, 44);
}