diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp index 56dbae75..b3c737fc 100644 --- a/src/files/WorldConverter.cpp +++ b/src/files/WorldConverter.cpp @@ -24,7 +24,7 @@ WorldConverter::WorldConverter( lut(lut), content(content) { - fs::path regionsFolder = wfile->getRegionsFolder(REGION_LAYER_VOXELS); + fs::path regionsFolder = wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS); if (!fs::is_directory(regionsFolder)) { logger.error() << "nothing to convert"; return; @@ -46,7 +46,7 @@ void WorldConverter::convertRegion(fs::path file) { return; } logger.info() << "converting region " << name; - wfile->processRegionVoxels(x, z, [=](ubyte* data) { + wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) { if (lut) { Chunk::convert(data, lut.get()); } diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index d9d2a598..a485b725 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -2,7 +2,6 @@ #include "../coders/byte_utils.h" #include "../coders/json.h" -#include "../coders/rle.h" #include "../constants.h" #include "../content/Content.h" #include "../core_defs.h" @@ -28,86 +27,20 @@ #include #include -#define REGION_FORMAT_MAGIC ".VOXREG" #define WORLD_FORMAT_MAGIC ".VOXWLD" const size_t BUFFER_SIZE_UNKNOWN = -1; -regfile::regfile(fs::path filename) : file(filename) { - if (file.length() < REGION_HEADER_SIZE) - throw std::runtime_error("incomplete region file header"); - char header[REGION_HEADER_SIZE]; - file.read(header, REGION_HEADER_SIZE); - - // avoid of use strcmp_s - if (std::string(header, strlen(REGION_FORMAT_MAGIC)) != REGION_FORMAT_MAGIC) { - throw std::runtime_error("invalid region file magic number"); - } - version = header[8]; - if (uint(version) > REGION_FORMAT_VERSION) { - throw illegal_region_format( - "region format "+std::to_string(version)+" is not supported" - ); - } +WorldFiles::WorldFiles(fs::path directory) : directory(directory), regions(directory) { } -WorldRegion::WorldRegion() { - chunksData = new ubyte*[REGION_CHUNKS_COUNT]{}; - sizes = new uint32_t[REGION_CHUNKS_COUNT]{}; -} - -WorldRegion::~WorldRegion() { - for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) { - delete[] chunksData[i]; - } - delete[] sizes; - delete[] chunksData; -} - -void WorldRegion::setUnsaved(bool unsaved) { - this->unsaved = unsaved; -} -bool WorldRegion::isUnsaved() const { - return unsaved; -} - -ubyte** WorldRegion::getChunks() const { - return chunksData; -} - -uint32_t* WorldRegion::getSizes() const { - return sizes; -} - -void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) { - size_t chunk_index = z * REGION_SIZE + x; - delete[] chunksData[chunk_index]; - chunksData[chunk_index] = data; - sizes[chunk_index] = size; -} - -ubyte* WorldRegion::getChunkData(uint x, uint z) { - return chunksData[z * REGION_SIZE + x]; -} - -uint WorldRegion::getChunkDataSize(uint x, uint z) { - return sizes[z * REGION_SIZE + x]; -} - -WorldFiles::WorldFiles(fs::path directory) : directory(directory) { - for (uint i = 0; i < sizeof(layers)/sizeof(RegionsLayer); i++) { - layers[i].layer = i; - } - layers[REGION_LAYER_VOXELS].folder = directory/fs::path("regions"); - layers[REGION_LAYER_LIGHTS].folder = directory/fs::path("lights"); - layers[REGION_LAYER_INVENTORIES].folder = directory/fs::path("inventories"); -} - -WorldFiles::WorldFiles(fs::path directory, const DebugSettings& settings) +WorldFiles::WorldFiles(fs::path directory, const DebugSettings& settings) : WorldFiles(directory) { generatorTestMode = settings.generatorTestMode; doWriteLights = settings.doWriteLights; + regions.generatorTestMode = generatorTestMode; + regions.doWriteLights = doWriteLights; } WorldFiles::~WorldFiles() { @@ -118,115 +51,6 @@ void WorldFiles::createDirectories() { fs::create_directories(directory / fs::path("content")); } -WorldRegion* WorldFiles::getRegion(int x, int z, int layer) { - RegionsLayer& regions = layers[layer]; - std::lock_guard lock(regions.mutex); - auto found = regions.regions.find(glm::ivec2(x, z)); - if (found == regions.regions.end()) - return nullptr; - return found->second.get(); -} - -WorldRegion* WorldFiles::getOrCreateRegion(int x, int z, int layer) { - RegionsLayer& regions = layers[layer]; - WorldRegion* region = getRegion(x, z, layer); - if (region == nullptr) { - std::lock_guard lock(regions.mutex); - region = new WorldRegion(); - regions.regions[glm::ivec2(x, z)].reset(region); - } - return region; -} - -std::unique_ptr WorldFiles::compress(const ubyte* src, size_t srclen, size_t& len) { - auto buffer = bufferPool.get(); - ubyte* bytes = buffer.get(); - - len = extrle::encode(src, srclen, bytes); - auto data = std::make_unique(len); - for (size_t i = 0; i < len; i++) { - data[i] = bytes[i]; - } - return data; -} - -std::unique_ptr WorldFiles::decompress(const ubyte* src, size_t srclen, size_t dstlen) { - auto decompressed = std::make_unique(dstlen); - extrle::decode(src, srclen, decompressed.get()); - return decompressed; -} - -inline void calc_reg_coords( - int x, int z, int& regionX, int& regionZ, int& localX, int& localZ -) { - regionX = floordiv(x, REGION_SIZE); - regionZ = floordiv(z, REGION_SIZE); - localX = x - (regionX * REGION_SIZE); - localZ = z - (regionZ * REGION_SIZE); -} - -void WorldFiles::put(int x, int z, int layer, std::unique_ptr data, size_t size, bool rle) { - if (rle) { - size_t compressedSize; - auto compressed = compress(data.get(), size, compressedSize); - put(x, z, layer, std::move(compressed), compressedSize, false); - return; - } - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); - region->setUnsaved(true); - region->put(localX, localZ, data.release(), size); -} - -std::unique_ptr write_inventories(Chunk* chunk, uint& datasize) { - auto& inventories = chunk->inventories; - ByteBuilder builder; - builder.putInt32(inventories.size()); - for (auto& entry : inventories) { - builder.putInt32(entry.first); - auto map = entry.second->serialize(); - auto bytes = json::to_binary(map.get(), true); - builder.putInt32(bytes.size()); - builder.put(bytes.data(), bytes.size()); - } - auto datavec = builder.data(); - datasize = builder.size(); - auto data = std::make_unique(datasize); - for (uint i = 0; i < datasize; i++) { - data[i] = datavec[i]; - } - return data; -} - -/// @brief Store chunk (voxels and lights) in region (existing or new) -void WorldFiles::put(Chunk* chunk){ - assert(chunk != nullptr); - - int regionX, regionZ, localX, localZ; - calc_reg_coords(chunk->x, chunk->z, regionX, regionZ, localX, localZ); - - put(chunk->x, chunk->z, REGION_LAYER_VOXELS, - chunk->encode(), CHUNK_DATA_LEN, true); - - // Writing lights cache - if (doWriteLights && chunk->isLighted()) { - put(chunk->x, chunk->z, REGION_LAYER_LIGHTS, - chunk->lightmap.encode(), LIGHTMAP_DATA_LEN, true); - } - // Writing block inventories - if (!chunk->inventories.empty()){ - uint datasize; - put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES, - write_inventories(chunk, datasize), datasize, false); - } -} - -fs::path WorldFiles::getRegionFilename(int x, int z) const { - return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); -} - /// @brief Extract X and Z from 'X_Z.bin' region file name. /// @param name source region file name /// @param x parsed X destination @@ -263,237 +87,7 @@ fs::path WorldFiles::getPacksFile() const { return directory/fs::path("packs.list"); } -std::unique_ptr WorldFiles::getChunk(int x, int z){ - uint32_t size; - auto* data = getData(x, z, REGION_LAYER_VOXELS, size); - if (data == nullptr) { - return nullptr; - } - return decompress(data, size, CHUNK_DATA_LEN); -} - -/// @brief Get cached lights for chunk at x,z -/// @return lights data or nullptr -std::unique_ptr WorldFiles::getLights(int x, int z) { - uint32_t size; - auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size); - if (bytes == nullptr) - return nullptr; - auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN); - return Lightmap::decode(data.get()); -} - -chunk_inventories_map WorldFiles::fetchInventories(int x, int z) { - chunk_inventories_map inventories; - uint32_t bytesSize; - const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize); - if (data == nullptr) - return inventories; - ByteReader reader(data, bytesSize); - int count = reader.getInt32(); - for (int i = 0; i < count; i++) { - uint index = reader.getInt32(); - uint size = reader.getInt32(); - auto map = json::from_binary(reader.pointer(), size); - reader.skip(size); - auto inv = std::make_shared(0, 0); - inv->deserialize(map.get()); - inventories[index] = inv; - } - return inventories; -} - -ubyte* WorldFiles::getData( - int x, int z, int layer, - uint32_t& size -) { - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - - WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); - ubyte* data = region->getChunkData(localX, localZ); - if (data == nullptr) { - auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer)); - if (regfile != nullptr) { - data = readChunkData(x, z, size, regfile.get()).release(); - } - if (data != nullptr) { - region->put(localX, localZ, data, size); - } - } - if (data != nullptr) { - size = region->getChunkDataSize(localX, localZ); - return data; - } - return nullptr; -} - -std::shared_ptr WorldFiles::useRegFile(glm::ivec3 coord) { - return std::shared_ptr(openRegFiles[coord].get(), [this](regfile* ptr) { - ptr->inUse = false; - regFilesCv.notify_one(); - }); -} - -void WorldFiles::closeRegFile(glm::ivec3 coord) { - openRegFiles.erase(coord); - regFilesCv.notify_one(); -} - -// Marks regfile as used and unmarks when shared_ptr dies -std::shared_ptr WorldFiles::getRegFile(glm::ivec3 coord) { - { - std::lock_guard lock(regFilesMutex); - const auto found = openRegFiles.find(coord); - if (found != openRegFiles.end()) { - if (found->second->inUse) { - throw std::runtime_error("regfile is currently in use"); - } - found->second->inUse = true; - return useRegFile(found->first); - } - } - return createRegFile(coord); -} - -std::shared_ptr WorldFiles::createRegFile(glm::ivec3 coord) { - fs::path file = layers[coord[2]].folder/getRegionFilename(coord[0], coord[1]); - if (!fs::exists(file)) { - return nullptr; - } - if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { - std::unique_lock lock(regFilesMutex); - while (true) { - bool closed = false; - // FIXME: bad choosing algorithm - for (auto& entry : openRegFiles) { - if (!entry.second->inUse) { - closeRegFile(entry.first); - closed = true; - break; - } - } - if (closed) { - break; - } - // notified when any regfile gets out of use or closed - regFilesCv.wait(lock); - } - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } else { - std::lock_guard lock(regFilesMutex); - openRegFiles[coord] = std::make_unique(file); - return useRegFile(coord); - } -} - -std::unique_ptr WorldFiles::readChunkData( - int x, - int z, - uint32_t& length, - regfile* rfile -){ - if (generatorTestMode) - return nullptr; - - int regionX, regionZ, localX, localZ; - calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - int chunkIndex = localZ * REGION_SIZE + localX; - - files::rafile& file = rfile->file; - size_t file_size = file.length(); - size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; - - uint32_t offset; - file.seekg(table_offset + chunkIndex * 4); - file.read((char*)(&offset), 4); - offset = dataio::read_int32_big((const ubyte*)(&offset), 0); - if (offset == 0){ - return nullptr; - } - - file.seekg(offset); - file.read((char*)(&offset), 4); - length = dataio::read_int32_big((const ubyte*)(&offset), 0); - auto data = std::make_unique(length); - file.read((char*)data.get(), length); - return data; -} - -/// @brief Read missing chunks data (null pointers) from region file -void WorldFiles::fetchChunks(WorldRegion* region, int x, int z, regfile* file) { - ubyte** chunks = region->getChunks(); - uint32_t* sizes = region->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; - int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; - if (chunks[i] == nullptr) { - chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file).release(); - } - } -} - -void WorldFiles::writeRegion(int x, int z, int layer, WorldRegion* entry){ - fs::path filename = layers[layer].folder/getRegionFilename(x, z); - - glm::ivec3 regcoord(x, z, layer); - if (auto regfile = getRegFile(regcoord)) { - fetchChunks(entry, x, z, regfile.get()); - - std::lock_guard lock(regFilesMutex); - closeRegFile(regcoord); - } - - char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; - header[8] = REGION_FORMAT_VERSION; - header[9] = 0; // flags - std::ofstream file(filename, std::ios::out | std::ios::binary); - file.write(header, REGION_HEADER_SIZE); - - size_t offset = REGION_HEADER_SIZE; - char intbuf[4]{}; - uint offsets[REGION_CHUNKS_COUNT]{}; - - ubyte** region = entry->getChunks(); - uint32_t* sizes = entry->getSizes(); - - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - ubyte* chunk = region[i]; - if (chunk == nullptr){ - offsets[i] = 0; - } else { - offsets[i] = offset; - - size_t compressedSize = sizes[i]; - dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0); - offset += 4 + compressedSize; - - file.write(intbuf, 4); - file.write((const char*)chunk, compressedSize); - } - } - for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { - dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0); - file.write(intbuf, 4); - } -} - -void WorldFiles::writeRegions(int layer) { - for (auto& it : layers[layer].regions){ - WorldRegion* region = it.second.get(); - if (region->getChunks() == nullptr || !region->isUnsaved()) - continue; - glm::ivec2 key = it.first; - writeRegion(key[0], key[1], layer, region); - } -} - void WorldFiles::write(const World* world, const Content* content) { - for (auto& layer : layers) { - fs::create_directories(layer.folder); - } if (world) { writeWorldInfo(world); if (!fs::exists(getPacksFile())) { @@ -505,9 +99,7 @@ void WorldFiles::write(const World* world, const Content* content) { } writeIndices(content->getIndices()); - for (auto& layer : layers) { - writeRegions(layer.layer); - } + regions.write(); } void WorldFiles::writePacks(const std::vector& packs) { @@ -585,34 +177,6 @@ void WorldFiles::removeIndices(const std::vector& packs) { files::write_json(getIndicesFile(), root.get()); } -void WorldFiles::processRegionVoxels(int x, int z, regionproc func) { - if (getRegion(x, z, REGION_LAYER_VOXELS)) { - throw std::runtime_error("not implemented for in-memory regions"); - } - auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS)); - if (regfile == nullptr) { - throw std::runtime_error("could not open region file"); - } - for (uint cz = 0; cz < REGION_SIZE; cz++) { - for (uint cx = 0; cx < REGION_SIZE; cx++) { - int gx = cx + x * REGION_SIZE; - int gz = cz + z * REGION_SIZE; - uint32_t length; - auto data = readChunkData(gx, gz, length, regfile.get()); - if (data == nullptr) - continue; - data = decompress(data.get(), length, CHUNK_DATA_LEN); - if (func(data.get())) { - put(gx, gz, REGION_LAYER_VOXELS, std::move(data), CHUNK_DATA_LEN, true); - } - } - } -} - fs::path WorldFiles::getFolder() const { return directory; } - -fs::path WorldFiles::getRegionsFolder(int layer) const { - return layers[layer].folder; -} diff --git a/src/files/WorldFiles.h b/src/files/WorldFiles.h index 2d49003f..7ef7e1e6 100644 --- a/src/files/WorldFiles.h +++ b/src/files/WorldFiles.h @@ -1,38 +1,24 @@ -#ifndef FILES_WORLDFILES_H_ -#define FILES_WORLDFILES_H_ +#ifndef FILES_WORLD_FILES_H_ +#define FILES_WORLD_FILES_H_ + +#include "WorldRegions.h" #include "files.h" #include "../typedefs.h" #include "../settings.h" #include "../content/ContentPack.h" #include "../voxels/Chunk.h" -#include "../util/BufferPool.h" -#include -#include #include #include #include -#include #include -#include #include #define GLM_ENABLE_EXPERIMENTAL #include "glm/gtx/hash.hpp" -inline constexpr uint REGION_HEADER_SIZE = 10; - -inline constexpr uint REGION_LAYER_VOXELS = 0; -inline constexpr uint REGION_LAYER_LIGHTS = 1; -inline constexpr uint REGION_LAYER_INVENTORIES = 2; - -inline constexpr uint REGION_SIZE_BIT = 5; -inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); -inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); -inline constexpr uint REGION_FORMAT_VERSION = 2; inline constexpr uint WORLD_FORMAT_VERSION = 1; -inline constexpr uint MAX_OPEN_REGION_FILES = 16; class Player; class Content; @@ -41,97 +27,19 @@ class World; namespace fs = std::filesystem; -class illegal_region_format : public std::runtime_error { -public: - illegal_region_format(const std::string& message) - : std::runtime_error(message) {} -}; - -class WorldRegion { - ubyte** chunksData; - uint32_t* sizes; - bool unsaved = false; -public: - WorldRegion(); - ~WorldRegion(); - - void put(uint x, uint z, ubyte* data, uint32_t size); - ubyte* getChunkData(uint x, uint z); - uint getChunkDataSize(uint x, uint z); - - void setUnsaved(bool unsaved); - bool isUnsaved() const; - - ubyte** getChunks() const; - uint32_t* getSizes() const; -}; - -struct regfile { - files::rafile file; - int version; - bool inUse = false; - - regfile(fs::path filename); -}; - -using regionsmap = std::unordered_map>; -using regionproc = std::function; - -struct RegionsLayer { - int layer; - fs::path folder; - regionsmap regions; - std::mutex mutex; -}; - class WorldFiles { fs::path directory; - std::unordered_map> openRegFiles; - std::mutex regFilesMutex; - std::condition_variable regFilesCv; - RegionsLayer layers[3] {}; + WorldRegions regions; + bool generatorTestMode = false; bool doWriteLights = true; - util::BufferPool bufferPool { - std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2 - }; - fs::path getRegionFilename(int x, int y) const; fs::path getWorldFile() const; fs::path getIndicesFile() const; fs::path getPacksFile() const; - - WorldRegion* getRegion(int x, int z, int layer); - WorldRegion* getOrCreateRegion(int x, int z, int layer); - - /// @brief Compress buffer with extrle - /// @param src source buffer - /// @param srclen length of the source buffer - /// @param len (out argument) length of result buffer - /// @return compressed bytes array - std::unique_ptr compress(const ubyte* src, size_t srclen, size_t& len); - - /// @brief Decompress buffer with extrle - /// @param src compressed buffer - /// @param srclen length of compressed buffer - /// @param dstlen max expected length of source buffer - /// @return decompressed bytes array - std::unique_ptr decompress(const ubyte* src, size_t srclen, size_t dstlen); - - std::unique_ptr readChunkData(int x, int y, uint32_t& length, regfile* file); - - void fetchChunks(WorldRegion* region, int x, int y, regfile* file); void writeWorldInfo(const World* world); - void writeRegions(int layer); void writeIndices(const ContentIndices* indices); - - ubyte* getData(int x, int z, int layer, uint32_t& size); - - std::shared_ptr getRegFile(glm::ivec3 coord); - void closeRegFile(glm::ivec3 coord); - std::shared_ptr useRegFile(glm::ivec3 coord); - std::shared_ptr createRegFile(glm::ivec3 coord); public: WorldFiles(fs::path directory); WorldFiles(fs::path directory, const DebugSettings& settings); @@ -140,30 +48,8 @@ public: fs::path getPlayerFile() const; void createDirectories(); - /// @brief Put all chunk data to regions - void put(Chunk* chunk); - - /// @brief Store data in specified region - /// @param x chunk.x - /// @param z chunk.z - /// @param layer regions layer - /// @param data target data - /// @param size data size - /// @param rle compress with ext-RLE - void put(int x, int z, int layer, std::unique_ptr data, size_t size, bool rle); - - std::unique_ptr getChunk(int x, int z); - std::unique_ptr getLights(int x, int z); - chunk_inventories_map fetchInventories(int x, int z); - bool readWorldInfo(World* world); - /// @brief Write or rewrite region file - /// @param x region X - /// @param z region Z - /// @param layer regions layer - void writeRegion(int x, int y, int layer, WorldRegion* entry); - /// @brief Write all unsaved data to world files /// @param world target world /// @param content world content @@ -173,14 +59,15 @@ public: void removeIndices(const std::vector& packs); - void processRegionVoxels(int x, int z, regionproc func); - /// @return world folder fs::path getFolder() const; - fs::path getRegionsFolder(int layer) const; static const inline std::string WORLD_FILE = "world.json"; static bool parseRegionFilename(const std::string& name, int& x, int& y); + + WorldRegions& getRegions() { + return regions; + } }; -#endif /* FILES_WORLDFILES_H_ */ +#endif // FILES_WORLD_FILES_H_ diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp new file mode 100644 index 00000000..111bf23e --- /dev/null +++ b/src/files/WorldRegions.cpp @@ -0,0 +1,456 @@ +#include "WorldRegions.h" + +#include "../coders/rle.h" +#include "../util/data_io.h" +#include "../coders/byte_utils.h" +#include "../maths/voxmaths.h" +#include "../items/Inventory.h" + +#include + +#define REGION_FORMAT_MAGIC ".VOXREG" + +regfile::regfile(fs::path filename) : file(filename) { + if (file.length() < REGION_HEADER_SIZE) + throw std::runtime_error("incomplete region file header"); + char header[REGION_HEADER_SIZE]; + file.read(header, REGION_HEADER_SIZE); + + // avoid of use strcmp_s + if (std::string(header, strlen(REGION_FORMAT_MAGIC)) != REGION_FORMAT_MAGIC) { + throw std::runtime_error("invalid region file magic number"); + } + version = header[8]; + if (uint(version) > REGION_FORMAT_VERSION) { + throw illegal_region_format( + "region format "+std::to_string(version)+" is not supported" + ); + } +} + +WorldRegion::WorldRegion() { + chunksData = new ubyte*[REGION_CHUNKS_COUNT]{}; + sizes = new uint32_t[REGION_CHUNKS_COUNT]{}; +} + +WorldRegion::~WorldRegion() { + for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) { + delete[] chunksData[i]; + } + delete[] sizes; + delete[] chunksData; +} + +void WorldRegion::setUnsaved(bool unsaved) { + this->unsaved = unsaved; +} +bool WorldRegion::isUnsaved() const { + return unsaved; +} + +ubyte** WorldRegion::getChunks() const { + return chunksData; +} + +uint32_t* WorldRegion::getSizes() const { + return sizes; +} + +void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) { + size_t chunk_index = z * REGION_SIZE + x; + delete[] chunksData[chunk_index]; + chunksData[chunk_index] = data; + sizes[chunk_index] = size; +} + +ubyte* WorldRegion::getChunkData(uint x, uint z) { + return chunksData[z * REGION_SIZE + x]; +} + +uint WorldRegion::getChunkDataSize(uint x, uint z) { + return sizes[z * REGION_SIZE + x]; +} + +WorldRegions::WorldRegions(fs::path directory) : directory(directory) { + for (uint i = 0; i < sizeof(layers)/sizeof(RegionsLayer); i++) { + layers[i].layer = i; + } + layers[REGION_LAYER_VOXELS].folder = directory/fs::path("regions"); + layers[REGION_LAYER_LIGHTS].folder = directory/fs::path("lights"); + layers[REGION_LAYER_INVENTORIES].folder = directory/fs::path("inventories"); +} + +WorldRegions::~WorldRegions() { +} + +WorldRegion* WorldRegions::getRegion(int x, int z, int layer) { + RegionsLayer& regions = layers[layer]; + std::lock_guard lock(regions.mutex); + auto found = regions.regions.find(glm::ivec2(x, z)); + if (found == regions.regions.end()) + return nullptr; + return found->second.get(); +} + +WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) { + RegionsLayer& regions = layers[layer]; + WorldRegion* region = getRegion(x, z, layer); + if (region == nullptr) { + std::lock_guard lock(regions.mutex); + region = new WorldRegion(); + regions.regions[glm::ivec2(x, z)].reset(region); + } + return region; +} + +std::unique_ptr WorldRegions::compress(const ubyte* src, size_t srclen, size_t& len) { + auto buffer = bufferPool.get(); + ubyte* bytes = buffer.get(); + + len = extrle::encode(src, srclen, bytes); + auto data = std::make_unique(len); + for (size_t i = 0; i < len; i++) { + data[i] = bytes[i]; + } + return data; +} + +std::unique_ptr WorldRegions::decompress(const ubyte* src, size_t srclen, size_t dstlen) { + auto decompressed = std::make_unique(dstlen); + extrle::decode(src, srclen, decompressed.get()); + return decompressed; +} + +inline void calc_reg_coords( + int x, int z, int& regionX, int& regionZ, int& localX, int& localZ +) { + regionX = floordiv(x, REGION_SIZE); + regionZ = floordiv(z, REGION_SIZE); + localX = x - (regionX * REGION_SIZE); + localZ = z - (regionZ * REGION_SIZE); +} + +std::unique_ptr WorldRegions::readChunkData( + int x, + int z, + uint32_t& length, + regfile* rfile +){ + if (generatorTestMode) + return nullptr; + + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + int chunkIndex = localZ * REGION_SIZE + localX; + + files::rafile& file = rfile->file; + size_t file_size = file.length(); + size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; + + uint32_t offset; + file.seekg(table_offset + chunkIndex * 4); + file.read((char*)(&offset), 4); + offset = dataio::read_int32_big((const ubyte*)(&offset), 0); + if (offset == 0){ + return nullptr; + } + + file.seekg(offset); + file.read((char*)(&offset), 4); + length = dataio::read_int32_big((const ubyte*)(&offset), 0); + auto data = std::make_unique(length); + file.read((char*)data.get(), length); + return data; +} + +/// @brief Read missing chunks data (null pointers) from region file +void WorldRegions::fetchChunks(WorldRegion* region, int x, int z, regfile* file) { + ubyte** chunks = region->getChunks(); + uint32_t* sizes = region->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE; + int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE; + if (chunks[i] == nullptr) { + chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file).release(); + } + } +} + +ubyte* WorldRegions::getData( + int x, int z, int layer, + uint32_t& size +) { + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + + WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); + ubyte* data = region->getChunkData(localX, localZ); + if (data == nullptr) { + auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer)); + if (regfile != nullptr) { + data = readChunkData(x, z, size, regfile.get()).release(); + } + if (data != nullptr) { + region->put(localX, localZ, data, size); + } + } + if (data != nullptr) { + size = region->getChunkDataSize(localX, localZ); + return data; + } + return nullptr; +} + +std::shared_ptr WorldRegions::useRegFile(glm::ivec3 coord) { + return std::shared_ptr(openRegFiles[coord].get(), [this](regfile* ptr) { + ptr->inUse = false; + regFilesCv.notify_one(); + }); +} + +void WorldRegions::closeRegFile(glm::ivec3 coord) { + openRegFiles.erase(coord); + regFilesCv.notify_one(); +} + +// Marks regfile as used and unmarks when shared_ptr dies +std::shared_ptr WorldRegions::getRegFile(glm::ivec3 coord) { + { + std::lock_guard lock(regFilesMutex); + const auto found = openRegFiles.find(coord); + if (found != openRegFiles.end()) { + if (found->second->inUse) { + throw std::runtime_error("regfile is currently in use"); + } + found->second->inUse = true; + return useRegFile(found->first); + } + } + return createRegFile(coord); +} + +std::shared_ptr WorldRegions::createRegFile(glm::ivec3 coord) { + fs::path file = layers[coord[2]].folder/getRegionFilename(coord[0], coord[1]); + if (!fs::exists(file)) { + return nullptr; + } + if (openRegFiles.size() == MAX_OPEN_REGION_FILES) { + std::unique_lock lock(regFilesMutex); + while (true) { + bool closed = false; + // FIXME: bad choosing algorithm + for (auto& entry : openRegFiles) { + if (!entry.second->inUse) { + closeRegFile(entry.first); + closed = true; + break; + } + } + if (closed) { + break; + } + // notified when any regfile gets out of use or closed + regFilesCv.wait(lock); + } + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } else { + std::lock_guard lock(regFilesMutex); + openRegFiles[coord] = std::make_unique(file); + return useRegFile(coord); + } +} + +fs::path WorldRegions::getRegionFilename(int x, int z) const { + return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin"); +} + +void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry){ + fs::path filename = layers[layer].folder/getRegionFilename(x, z); + + glm::ivec3 regcoord(x, z, layer); + if (auto regfile = getRegFile(regcoord)) { + fetchChunks(entry, x, z, regfile.get()); + + std::lock_guard lock(regFilesMutex); + closeRegFile(regcoord); + } + + char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC; + header[8] = REGION_FORMAT_VERSION; + header[9] = 0; // flags + std::ofstream file(filename, std::ios::out | std::ios::binary); + file.write(header, REGION_HEADER_SIZE); + + size_t offset = REGION_HEADER_SIZE; + char intbuf[4]{}; + uint offsets[REGION_CHUNKS_COUNT]{}; + + ubyte** region = entry->getChunks(); + uint32_t* sizes = entry->getSizes(); + + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + ubyte* chunk = region[i]; + if (chunk == nullptr){ + offsets[i] = 0; + } else { + offsets[i] = offset; + + size_t compressedSize = sizes[i]; + dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0); + offset += 4 + compressedSize; + + file.write(intbuf, 4); + file.write((const char*)chunk, compressedSize); + } + } + for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) { + dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0); + file.write(intbuf, 4); + } +} + +void WorldRegions::writeRegions(int layer) { + for (auto& it : layers[layer].regions){ + WorldRegion* region = it.second.get(); + if (region->getChunks() == nullptr || !region->isUnsaved()) + continue; + glm::ivec2 key = it.first; + writeRegion(key[0], key[1], layer, region); + } +} + +void WorldRegions::put(int x, int z, int layer, std::unique_ptr data, size_t size, bool rle) { + if (rle) { + size_t compressedSize; + auto compressed = compress(data.get(), size, compressedSize); + put(x, z, layer, std::move(compressed), compressedSize, false); + return; + } + int regionX, regionZ, localX, localZ; + calc_reg_coords(x, z, regionX, regionZ, localX, localZ); + + WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer); + region->setUnsaved(true); + region->put(localX, localZ, data.release(), size); +} + +static std::unique_ptr write_inventories(Chunk* chunk, uint& datasize) { + auto& inventories = chunk->inventories; + ByteBuilder builder; + builder.putInt32(inventories.size()); + for (auto& entry : inventories) { + builder.putInt32(entry.first); + auto map = entry.second->serialize(); + auto bytes = json::to_binary(map.get(), true); + builder.putInt32(bytes.size()); + builder.put(bytes.data(), bytes.size()); + } + auto datavec = builder.data(); + datasize = builder.size(); + auto data = std::make_unique(datasize); + for (uint i = 0; i < datasize; i++) { + data[i] = datavec[i]; + } + return data; +} + +/// @brief Store chunk (voxels and lights) in region (existing or new) +void WorldRegions::put(Chunk* chunk){ + assert(chunk != nullptr); + + int regionX, regionZ, localX, localZ; + calc_reg_coords(chunk->x, chunk->z, regionX, regionZ, localX, localZ); + + put(chunk->x, chunk->z, REGION_LAYER_VOXELS, + chunk->encode(), CHUNK_DATA_LEN, true); + + // Writing lights cache + if (doWriteLights && chunk->isLighted()) { + put(chunk->x, chunk->z, REGION_LAYER_LIGHTS, + chunk->lightmap.encode(), LIGHTMAP_DATA_LEN, true); + } + // Writing block inventories + if (!chunk->inventories.empty()){ + uint datasize; + put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES, + write_inventories(chunk, datasize), datasize, false); + } +} + +std::unique_ptr WorldRegions::getChunk(int x, int z){ + uint32_t size; + auto* data = getData(x, z, REGION_LAYER_VOXELS, size); + if (data == nullptr) { + return nullptr; + } + return decompress(data, size, CHUNK_DATA_LEN); +} + +/// @brief Get cached lights for chunk at x,z +/// @return lights data or nullptr +std::unique_ptr WorldRegions::getLights(int x, int z) { + uint32_t size; + auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size); + if (bytes == nullptr) + return nullptr; + auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN); + return Lightmap::decode(data.get()); +} + +chunk_inventories_map WorldRegions::fetchInventories(int x, int z) { + chunk_inventories_map inventories; + uint32_t bytesSize; + const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize); + if (data == nullptr) + return inventories; + ByteReader reader(data, bytesSize); + int count = reader.getInt32(); + for (int i = 0; i < count; i++) { + uint index = reader.getInt32(); + uint size = reader.getInt32(); + auto map = json::from_binary(reader.pointer(), size); + reader.skip(size); + auto inv = std::make_shared(0, 0); + inv->deserialize(map.get()); + inventories[index] = inv; + } + return inventories; +} + +void WorldRegions::processRegionVoxels(int x, int z, regionproc func) { + if (getRegion(x, z, REGION_LAYER_VOXELS)) { + throw std::runtime_error("not implemented for in-memory regions"); + } + auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS)); + if (regfile == nullptr) { + throw std::runtime_error("could not open region file"); + } + for (uint cz = 0; cz < REGION_SIZE; cz++) { + for (uint cx = 0; cx < REGION_SIZE; cx++) { + int gx = cx + x * REGION_SIZE; + int gz = cz + z * REGION_SIZE; + uint32_t length; + auto data = readChunkData(gx, gz, length, regfile.get()); + if (data == nullptr) + continue; + data = decompress(data.get(), length, CHUNK_DATA_LEN); + if (func(data.get())) { + put(gx, gz, REGION_LAYER_VOXELS, std::move(data), CHUNK_DATA_LEN, true); + } + } + } +} + +fs::path WorldRegions::getRegionsFolder(int layer) const { + return layers[layer].folder; +} + + +void WorldRegions::write() { + for (auto& layer : layers) { + fs::create_directories(layer.folder); + writeRegions(layer.layer); + } +} diff --git a/src/files/WorldRegions.h b/src/files/WorldRegions.h new file mode 100644 index 00000000..98d7a027 --- /dev/null +++ b/src/files/WorldRegions.h @@ -0,0 +1,155 @@ +#ifndef FILES_WORLD_REGIONS_H_ +#define FILES_WORLD_REGIONS_H_ + +#include "files.h" +#include "../typedefs.h" +#include "../util/BufferPool.h" +#include "../voxels/Chunk.h" + +#include +#include +#include +#include +#include +#include + +#include +#define GLM_ENABLE_EXPERIMENTAL +#include "glm/gtx/hash.hpp" + +namespace fs = std::filesystem; + +inline constexpr uint REGION_HEADER_SIZE = 10; + +inline constexpr uint REGION_LAYER_VOXELS = 0; +inline constexpr uint REGION_LAYER_LIGHTS = 1; +inline constexpr uint REGION_LAYER_INVENTORIES = 2; + +inline constexpr uint REGION_SIZE_BIT = 5; +inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT)); +inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE)); +inline constexpr uint REGION_FORMAT_VERSION = 2; +inline constexpr uint MAX_OPEN_REGION_FILES = 16; + +class illegal_region_format : public std::runtime_error { +public: + illegal_region_format(const std::string& message) + : std::runtime_error(message) {} +}; + +class WorldRegion { + ubyte** chunksData; + uint32_t* sizes; + bool unsaved = false; +public: + WorldRegion(); + ~WorldRegion(); + + void put(uint x, uint z, ubyte* data, uint32_t size); + ubyte* getChunkData(uint x, uint z); + uint getChunkDataSize(uint x, uint z); + + void setUnsaved(bool unsaved); + bool isUnsaved() const; + + ubyte** getChunks() const; + uint32_t* getSizes() const; +}; + +struct regfile { + files::rafile file; + int version; + bool inUse = false; + + regfile(fs::path filename); +}; + +using regionsmap = std::unordered_map>; +using regionproc = std::function; + +struct RegionsLayer { + int layer; + fs::path folder; + regionsmap regions; + std::mutex mutex; +}; + +class WorldRegions { + fs::path directory; + std::unordered_map> openRegFiles; + std::mutex regFilesMutex; + std::condition_variable regFilesCv; + RegionsLayer layers[3] {}; + util::BufferPool bufferPool { + std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2 + }; + + WorldRegion* getRegion(int x, int z, int layer); + WorldRegion* getOrCreateRegion(int x, int z, int layer); + + /// @brief Compress buffer with extrle + /// @param src source buffer + /// @param srclen length of the source buffer + /// @param len (out argument) length of result buffer + /// @return compressed bytes array + std::unique_ptr compress(const ubyte* src, size_t srclen, size_t& len); + + /// @brief Decompress buffer with extrle + /// @param src compressed buffer + /// @param srclen length of compressed buffer + /// @param dstlen max expected length of source buffer + /// @return decompressed bytes array + std::unique_ptr decompress(const ubyte* src, size_t srclen, size_t dstlen); + + std::unique_ptr readChunkData(int x, int y, uint32_t& length, regfile* file); + + void fetchChunks(WorldRegion* region, int x, int y, regfile* file); + + ubyte* getData(int x, int z, int layer, uint32_t& size); + + std::shared_ptr getRegFile(glm::ivec3 coord); + void closeRegFile(glm::ivec3 coord); + std::shared_ptr useRegFile(glm::ivec3 coord); + std::shared_ptr createRegFile(glm::ivec3 coord); + + fs::path getRegionFilename(int x, int y) const; + + void writeRegions(int layer); + + /// @brief Write or rewrite region file + /// @param x region X + /// @param z region Z + /// @param layer regions layer + void writeRegion(int x, int y, int layer, WorldRegion* entry); +public: + bool generatorTestMode = false; + bool doWriteLights = true; + + WorldRegions(fs::path directory); + WorldRegions(const WorldRegions&) = delete; + ~WorldRegions(); + + /// @brief Put all chunk data to regions + void put(Chunk* chunk); + + /// @brief Store data in specified region + /// @param x chunk.x + /// @param z chunk.z + /// @param layer regions layer + /// @param data target data + /// @param size data size + /// @param rle compress with ext-RLE + void put(int x, int z, int layer, std::unique_ptr data, size_t size, bool rle); + + std::unique_ptr getChunk(int x, int z); + std::unique_ptr getLights(int x, int z); + chunk_inventories_map fetchInventories(int x, int z); + + void processRegionVoxels(int x, int z, regionproc func); + + fs::path getRegionsFolder(int layer) const; + + void write(); +}; + +#endif // FILES_WORLD_REGIONS_H_ diff --git a/src/voxels/Chunks.cpp b/src/voxels/Chunks.cpp index af5a0ae3..2f2323ed 100644 --- a/src/voxels/Chunks.cpp +++ b/src/voxels/Chunks.cpp @@ -410,6 +410,7 @@ void Chunks::setCenter(int32_t x, int32_t z) { } void Chunks::translate(int32_t dx, int32_t dz) { + auto& regions = worldFiles->getRegions(); for (uint i = 0; i < volume; i++){ chunksSecond[i] = nullptr; } @@ -422,8 +423,9 @@ void Chunks::translate(int32_t dx, int32_t dz) { continue; if (nx < 0 || nz < 0 || nx >= int(w) || nz >= int(d)){ events->trigger(EVT_CHUNK_HIDDEN, chunk.get()); - if (worldFiles) - worldFiles->put(chunk.get()); + if (worldFiles) { + regions.put(chunk.get()); + } chunksCount--; continue; } @@ -482,10 +484,11 @@ bool Chunks::putChunk(std::shared_ptr chunk) { } void Chunks::saveAndClear(){ + auto& regions = worldFiles->getRegions(); for (size_t i = 0; i < volume; i++){ Chunk* chunk = chunks[i].get(); if (chunk) { - worldFiles->put(chunk); + regions.put(chunk); events->trigger(EVT_CHUNK_HIDDEN, chunk); } chunks[i] = nullptr; diff --git a/src/voxels/ChunksStorage.cpp b/src/voxels/ChunksStorage.cpp index 69de518c..35484a3f 100644 --- a/src/voxels/ChunksStorage.cpp +++ b/src/voxels/ChunksStorage.cpp @@ -51,14 +51,14 @@ static void verifyLoadedChunk(ContentIndices* indices, Chunk* chunk) { std::shared_ptr ChunksStorage::create(int x, int z) { World* world = level->getWorld(); - WorldFiles* wfile = world->wfile.get(); + auto& regions = world->wfile.get()->getRegions(); auto chunk = std::make_shared(x, z); store(chunk); - auto data = wfile->getChunk(chunk->x, chunk->z); + auto data = regions.getChunk(chunk->x, chunk->z); if (data) { chunk->decode(data.get()); - auto invs = wfile->fetchInventories(chunk->x, chunk->z); + auto invs = regions.fetchInventories(chunk->x, chunk->z); chunk->setBlockInventories(std::move(invs)); chunk->setLoaded(true); for(auto& entry : chunk->inventories) { @@ -67,7 +67,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { verifyLoadedChunk(level->content->getIndices(), chunk.get()); } - auto lights = wfile->getLights(chunk->x, chunk->z); + auto lights = regions.getLights(chunk->x, chunk->z); if (lights) { chunk->lightmap.set(lights.get()); chunk->setLoadedLights(true); diff --git a/src/world/World.cpp b/src/world/World.cpp index 800518df..2a280c9a 100644 --- a/src/world/World.cpp +++ b/src/world/World.cpp @@ -49,8 +49,8 @@ void World::updateTimers(float delta) { void World::write(Level* level) { const Content* content = level->content; - Chunks* chunks = level->chunks.get(); + auto& regions = wfile->getRegions(); for (size_t i = 0; i < chunks->volume; i++) { auto chunk = chunks->chunks[i]; @@ -60,7 +60,7 @@ void World::write(Level* level) { settings.debug.doWriteLights; if (!chunk->isUnsaved() && !lightsUnsaved) continue; - wfile->put(chunk.get()); + regions.put(chunk.get()); } wfile->write(this, content);