diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp new file mode 100644 index 00000000..d32690a2 --- /dev/null +++ b/src/coders/compression.cpp @@ -0,0 +1,87 @@ +#include "compression.hpp" + +#include +#include + +#include "rle.hpp" +#include "gzip.hpp" +#include "util/BufferPool.hpp" + +using namespace compression; + +static util::BufferPool buffer_pools[] { + {255}, + {UINT16_MAX}, + {UINT16_MAX * 8}, +}; + +static std::shared_ptr get_buffer(size_t minSize) { + for (auto& pool : buffer_pools) { + if (minSize <= pool.getBufferSize()) { + return pool.get(); + } + } + return nullptr; +} + +std::unique_ptr compression::compress( + const ubyte* src, size_t srclen, size_t& len, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: { + // max extrle out size is srcLen * 2 + auto buffer = get_buffer(srclen * 2); + auto bytes = buffer.get(); + std::unique_ptr uptr; + if (bytes == nullptr) { + uptr = std::make_unique(srclen * 2); + bytes = uptr.get(); + } + len = extrle::encode(src, srclen, bytes); + if (uptr) { + return uptr; + } + auto data = std::make_unique(len); + std::memcpy(data.get(), bytes, len); + return data; + } + case Method::GZIP: { + auto buffer = gzip::compress(src, srclen); + auto data = std::make_unique(buffer.size()); + std::memcpy(data.get(), buffer.data(), buffer.size()); + len = buffer.size(); + return data; + } + default: + throw std::runtime_error("not implemented"); + } +} + +std::unique_ptr compression::decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method +) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: { + auto decompressed = std::make_unique(dstlen); + extrle::decode(src, srclen, decompressed.get()); + return decompressed; + } + case Method::GZIP: { + auto buffer = gzip::decompress(src, srclen); + if (buffer.size() != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(buffer.size())); + } + auto decompressed = std::make_unique(buffer.size()); + std::memcpy(decompressed.get(), buffer.data(), buffer.size()); + return decompressed; + } + default: + throw std::runtime_error("not implemented"); + } +} diff --git a/src/coders/compression.hpp b/src/coders/compression.hpp new file mode 100644 index 00000000..09f50b2a --- /dev/null +++ b/src/coders/compression.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "typedefs.hpp" + +namespace compression { + enum class Method { + NONE, EXTRLE8, GZIP + }; + + /// @brief Compress buffer + /// @param src source buffer + /// @param srclen length of the source buffer + /// @param len (out argument) length of result buffer + /// @param method compression method + /// @return compressed bytes array + /// @throws std::invalid_argument if compression method is NONE + std::unique_ptr compress( + const ubyte* src, size_t srclen, size_t& len, Method method + ); + + /// @brief Decompress buffer + /// @param src compressed buffer + /// @param srclen length of compressed buffer + /// @param dstlen max expected length of source buffer + /// @return decompressed bytes array + std::unique_ptr decompress( + const ubyte* src, size_t srclen, size_t dstlen, Method method); +} diff --git a/src/files/RegionsLayer.cpp b/src/files/RegionsLayer.cpp index 2fc405de..2ec8782a 100644 --- a/src/files/RegionsLayer.cpp +++ b/src/files/RegionsLayer.cpp @@ -11,7 +11,7 @@ static fs::path get_region_filename(int x, int z) { } /// @brief Read missing chunks data (null pointers) from region file -static void fetchChunks(WorldRegion* region, int x, int z, regfile* file) { +static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) { auto* chunks = region->getChunks(); uint32_t* sizes = region->getSizes(); @@ -174,7 +174,7 @@ void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) { glm::ivec2 regcoord(x, z); if (auto regfile = getRegFile(regcoord, false)) { - fetchChunks(entry, x, z, regfile.get()); + fetch_chunks(entry, x, z, regfile.get()); std::lock_guard lock(regFilesMutex); regfile.reset(); diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp index 9074c69e..07400895 100644 --- a/src/files/WorldRegions.cpp +++ b/src/files/WorldRegions.cpp @@ -5,7 +5,6 @@ #include #include "coders/byte_utils.hpp" -#include "coders/rle.hpp" #include "data/dynamic.hpp" #include "items/Inventory.hpp" #include "maths/voxmaths.hpp" @@ -55,8 +54,14 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) { layers[i].layer = i; } - layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions"); - layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights"); + auto& voxels = layers[REGION_LAYER_VOXELS]; + voxels.folder = directory / fs::path("regions"); + voxels.compression = compression::Method::EXTRLE8; + + auto& lights = layers[REGION_LAYER_LIGHTS]; + lights.folder = directory / fs::path("lights"); + lights.compression = compression::Method::EXTRLE8; + layers[REGION_LAYER_INVENTORIES].folder = directory / fs::path("inventories"); layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities"); @@ -64,28 +69,6 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) { WorldRegions::~WorldRegions() = default; -std::unique_ptr WorldRegions::compress( - const ubyte* src, size_t srclen, size_t& len -) { - auto buffer = bufferPool.get(); - auto bytes = buffer.get(); - - len = extrle::encode(src, srclen, bytes); - auto data = std::make_unique(len); - for (size_t i = 0; i < len; i++) { - data[i] = bytes[i]; - } - return data; -} - -std::unique_ptr WorldRegions::decompress( - const ubyte* src, size_t srclen, size_t dstlen -) { - auto decompressed = std::make_unique(dstlen); - extrle::decode(src, srclen, decompressed.get()); - return decompressed; -} - void RegionsLayer::writeAll() { for (auto& it : regions) { WorldRegion* region = it.second.get(); @@ -100,21 +83,19 @@ void RegionsLayer::writeAll() { void WorldRegions::put( int x, int z, - int layer, + int layerid, std::unique_ptr data, - size_t size, - bool rle + size_t size ) { - if (rle) { - size_t compressedSize; - auto compressed = compress(data.get(), size, compressedSize); - put(x, z, layer, std::move(compressed), compressedSize, false); - return; + auto& layer = layers[layerid]; + if (layer.compression != compression::Method::NONE) { + data = compression::compress( + data.get(), size, size, layer.compression); } int regionX, regionZ, localX, localZ; calc_reg_coords(x, z, regionX, regionZ, localX, localZ); - WorldRegion* region = layers[layer].getOrCreateRegion(regionX, regionZ); + WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ); region->setUnsaved(true); region->put(localX, localZ, std::move(data), size); } @@ -139,7 +120,6 @@ static std::unique_ptr write_inventories( return data; } -/// @brief Store chunk data (voxels and lights) in region (existing or new) void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { assert(chunk != nullptr); if (!chunk->flags.lighted) { @@ -157,8 +137,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_VOXELS, chunk->encode(), - CHUNK_DATA_LEN, - true); + CHUNK_DATA_LEN); // Writing lights cache if (doWriteLights && chunk->flags.lighted) { @@ -166,8 +145,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_LIGHTS, chunk->lightmap.encode(), - LIGHTMAP_DATA_LEN, - true); + LIGHTMAP_DATA_LEN); } // Writing block inventories if (!chunk->inventories.empty()) { @@ -177,8 +155,7 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_INVENTORIES, std::move(data), - datasize, - false); + datasize); } // Writing entities if (!entitiesData.empty()) { @@ -190,29 +167,30 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { chunk->z, REGION_LAYER_ENTITIES, std::move(data), - entitiesData.size(), - false); + entitiesData.size()); } } std::unique_ptr WorldRegions::getVoxels(int x, int z) { uint32_t size; - auto* data = layers[REGION_LAYER_VOXELS].getData(x, z, size); + auto& layer = layers[REGION_LAYER_VOXELS]; + auto* data = layer.getData(x, z, size); if (data == nullptr) { return nullptr; } - return decompress(data, size, CHUNK_DATA_LEN); + return compression::decompress(data, size, CHUNK_DATA_LEN, layer.compression); } -/// @brief Get cached lights for chunk at x,z -/// @return lights data or nullptr std::unique_ptr WorldRegions::getLights(int x, int z) { uint32_t size; - auto* bytes = layers[REGION_LAYER_LIGHTS].getData(x, z, size); + auto& layer = layers[REGION_LAYER_LIGHTS]; + auto* bytes = layer.getData(x, z, size); if (bytes == nullptr) { return nullptr; } - auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN); + auto data = compression::decompress( + bytes, size, LIGHTMAP_DATA_LEN, layer.compression + ); return Lightmap::decode(data.get()); } @@ -254,10 +232,11 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) { } void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { - if (layers[REGION_LAYER_VOXELS].getRegion(x, z)) { + auto& layer = layers[REGION_LAYER_VOXELS]; + if (layer.getRegion(x, z)) { throw std::runtime_error("not implemented for in-memory regions"); } - auto regfile = layers[REGION_LAYER_VOXELS].getRegFile({x, z}); + auto regfile = layer.getRegFile({x, z}); if (regfile == nullptr) { throw std::runtime_error("could not open region file"); } @@ -271,14 +250,15 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) { if (data == nullptr) { continue; } - data = decompress(data.get(), length, CHUNK_DATA_LEN); + data = compression::decompress( + data.get(), length, CHUNK_DATA_LEN, layer.compression + ); if (func(data.get())) { put(gx, gz, REGION_LAYER_VOXELS, std::move(data), - CHUNK_DATA_LEN, - true); + CHUNK_DATA_LEN); } } } diff --git a/src/files/WorldRegions.hpp b/src/files/WorldRegions.hpp index e467b798..df77fdb2 100644 --- a/src/files/WorldRegions.hpp +++ b/src/files/WorldRegions.hpp @@ -8,11 +8,12 @@ #include #include -#include "data/dynamic_fwd.hpp" #include "typedefs.hpp" +#include "data/dynamic_fwd.hpp" #include "util/BufferPool.hpp" #include "voxels/Chunk.hpp" #include "maths/voxmaths.hpp" +#include "coders/compression.hpp" #include "files.hpp" #define GLM_ENABLE_EXPERIMENTAL @@ -126,6 +127,8 @@ struct RegionsLayer { /// @brief Regions layer folder fs::path folder; + compression::Method compression = compression::Method::NONE; + /// @brief In-memory regions data regionsmap regions; @@ -139,7 +142,7 @@ struct RegionsLayer { std::mutex regFilesMutex; std::condition_variable regFilesCv; - regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); + [[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true); [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord); regfile_ptr createRegFile(glm::ivec2 coord); void closeRegFile(glm::ivec2 coord); @@ -178,26 +181,6 @@ class WorldRegions { fs::path directory; RegionsLayer layers[4] {}; - util::BufferPool bufferPool { - std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2}; - - /// @brief Compress buffer with extrle - /// @param src source buffer - /// @param srclen length of the source buffer - /// @param len (out argument) length of result buffer - /// @return compressed bytes array - std::unique_ptr compress( - const ubyte* src, size_t srclen, size_t& len - ); - - /// @brief Decompress buffer with extrle - /// @param src compressed buffer - /// @param srclen length of compressed buffer - /// @param dstlen max expected length of source buffer - /// @return decompressed bytes array - std::unique_ptr decompress( - const ubyte* src, size_t srclen, size_t dstlen - ); public: bool generatorTestMode = false; bool doWriteLights = true; @@ -215,19 +198,22 @@ public: /// @param layer regions layer /// @param data target data /// @param size data size - /// @param rle compress with ext-RLE void put( int x, int z, int layer, std::unique_ptr data, - size_t size, - bool rle + size_t size ); std::unique_ptr getVoxels(int x, int z); + + /// @brief Get cached lights for chunk at x,z + /// @return lights data or nullptr std::unique_ptr getLights(int x, int z); + chunk_inventories_map fetchInventories(int x, int z); + dynamic::Map_sptr fetchEntities(int x, int z); void processRegionVoxels(int x, int z, const regionproc& func); diff --git a/src/util/BufferPool.hpp b/src/util/BufferPool.hpp index 0d775b90..aeeebb8c 100644 --- a/src/util/BufferPool.hpp +++ b/src/util/BufferPool.hpp @@ -35,5 +35,9 @@ namespace util { freeBuffers.push(ptr); }); } + + size_t getBufferSize() const { + return bufferSize; + } }; } diff --git a/src/voxels/ChunksStorage.cpp b/src/voxels/ChunksStorage.cpp index 23665b61..b430dbe4 100644 --- a/src/voxels/ChunksStorage.cpp +++ b/src/voxels/ChunksStorage.cpp @@ -59,8 +59,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { auto chunk = std::make_shared(x, z); store(chunk); - auto data = regions.getVoxels(chunk->x, chunk->z); - if (data) { + if (auto data = regions.getVoxels(chunk->x, chunk->z)) { chunk->decode(data.get()); auto invs = regions.fetchInventories(chunk->x, chunk->z); @@ -77,9 +76,7 @@ std::shared_ptr ChunksStorage::create(int x, int z) { } verifyLoadedChunk(level->content->getIndices(), chunk.get()); } - - auto lights = regions.getLights(chunk->x, chunk->z); - if (lights) { + if (auto lights = regions.getLights(chunk->x, chunk->z)) { chunk->lightmap.set(lights.get()); chunk->flags.loadedLights = true; }