PulseStudio/VoxelEngine
PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

add /coders/compression module

Browse Source
This commit is contained in:
MihailRis 2024-09-01 21:58:33 +03:00
parent 0f53d5b835
commit 7d193941a4
7 changed files with 170 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/coders/compression.cpp Normal file
View File

@ -0,0 +1,87 @@
#include "compression.hpp"
#include <cstring>
#include <stdexcept>
#include "rle.hpp"
#include "gzip.hpp"
#include "util/BufferPool.hpp"
using namespace compression;
static util::BufferPool<ubyte> buffer_pools[] {
{255},
{UINT16_MAX},
{UINT16_MAX * 8},
};
static std::shared_ptr<ubyte[]> get_buffer(size_t minSize) {
for (auto& pool : buffer_pools) {
if (minSize <= pool.getBufferSize()) {
return pool.get();
}
}
return nullptr;
}
std::unique_ptr<ubyte[]> compression::compress(
const ubyte* src, size_t srclen, size_t& len, Method method
) {
switch (method) {
case Method::NONE:
throw std::invalid_argument("compression method is NONE");
case Method::EXTRLE8: {
// max extrle out size is srcLen * 2
auto buffer = get_buffer(srclen * 2);
auto bytes = buffer.get();
std::unique_ptr<ubyte[]> uptr;
if (bytes == nullptr) {
uptr = std::make_unique<ubyte[]>(srclen * 2);
bytes = uptr.get();
}
len = extrle::encode(src, srclen, bytes);
if (uptr) {
return uptr;
}
auto data = std::make_unique<ubyte[]>(len);
std::memcpy(data.get(), bytes, len);
return data;
}
case Method::GZIP: {
auto buffer = gzip::compress(src, srclen);
auto data = std::make_unique<ubyte[]>(buffer.size());
std::memcpy(data.get(), buffer.data(), buffer.size());
len = buffer.size();
return data;
}
default:
throw std::runtime_error("not implemented");
}
}
std::unique_ptr<ubyte[]> compression::decompress(
const ubyte* src, size_t srclen, size_t dstlen, Method method
) {
switch (method) {
case Method::NONE:
throw std::invalid_argument("compression method is NONE");
case Method::EXTRLE8: {
auto decompressed = std::make_unique<ubyte[]>(dstlen);
extrle::decode(src, srclen, decompressed.get());
return decompressed;
}
case Method::GZIP: {
auto buffer = gzip::decompress(src, srclen);
if (buffer.size() != dstlen) {
throw std::runtime_error(
"expected decompressed size " + std::to_string(dstlen) +
" got " + std::to_string(buffer.size()));
}
auto decompressed = std::make_unique<ubyte[]>(buffer.size());
std::memcpy(decompressed.get(), buffer.data(), buffer.size());
return decompressed;
}
default:
throw std::runtime_error("not implemented");
}
}

30
src/coders/compression.hpp Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#include <memory>
#include "typedefs.hpp"
namespace compression {
enum class Method {
NONE, EXTRLE8, GZIP
};
/// @brief Compress buffer
/// @param src source buffer
/// @param srclen length of the source buffer
/// @param len (out argument) length of result buffer
/// @param method compression method
/// @return compressed bytes array
/// @throws std::invalid_argument if compression method is NONE
std::unique_ptr<ubyte[]> compress(
const ubyte* src, size_t srclen, size_t& len, Method method
);
/// @brief Decompress buffer
/// @param src compressed buffer
/// @param srclen length of compressed buffer
/// @param dstlen max expected length of source buffer
/// @return decompressed bytes array
std::unique_ptr<ubyte[]> decompress(
const ubyte* src, size_t srclen, size_t dstlen, Method method);
}

4
src/files/RegionsLayer.cpp
View File

@ -11,7 +11,7 @@ static fs::path get_region_filename(int x, int z) {
}
/// @brief Read missing chunks data (null pointers) from region file
static void fetchChunks(WorldRegion* region, int x, int z, regfile* file) {
static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) {
auto* chunks = region->getChunks();
uint32_t* sizes = region->getSizes();
@ -174,7 +174,7 @@ void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) {
glm::ivec2 regcoord(x, z);
if (auto regfile = getRegFile(regcoord, false)) {
fetchChunks(entry, x, z, regfile.get());
fetch_chunks(entry, x, z, regfile.get());
std::lock_guard lock(regFilesMutex);
regfile.reset();

88
src/files/WorldRegions.cpp
View File

@ -5,7 +5,6 @@
#include <vector>
#include "coders/byte_utils.hpp"
#include "coders/rle.hpp"
#include "data/dynamic.hpp"
#include "items/Inventory.hpp"
#include "maths/voxmaths.hpp"
@ -55,8 +54,14 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) {
layers[i].layer = i;
}
layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions");
layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights");
auto& voxels = layers[REGION_LAYER_VOXELS];
voxels.folder = directory / fs::path("regions");
voxels.compression = compression::Method::EXTRLE8;
auto& lights = layers[REGION_LAYER_LIGHTS];
lights.folder = directory / fs::path("lights");
lights.compression = compression::Method::EXTRLE8;
layers[REGION_LAYER_INVENTORIES].folder =
directory / fs::path("inventories");
layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities");
@ -64,28 +69,6 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
WorldRegions::~WorldRegions() = default;
std::unique_ptr<ubyte[]> WorldRegions::compress(
const ubyte* src, size_t srclen, size_t& len
) {
auto buffer = bufferPool.get();
auto bytes = buffer.get();
len = extrle::encode(src, srclen, bytes);
auto data = std::make_unique<ubyte[]>(len);
for (size_t i = 0; i < len; i++) {
data[i] = bytes[i];
}
return data;
}
std::unique_ptr<ubyte[]> WorldRegions::decompress(
const ubyte* src, size_t srclen, size_t dstlen
) {
auto decompressed = std::make_unique<ubyte[]>(dstlen);
extrle::decode(src, srclen, decompressed.get());
return decompressed;
}
void RegionsLayer::writeAll() {
for (auto& it : regions) {
WorldRegion* region = it.second.get();
@ -100,21 +83,19 @@ void RegionsLayer::writeAll() {
void WorldRegions::put(
int x,
int z,
int layer,
int layerid,
std::unique_ptr<ubyte[]> data,
size_t size,
bool rle
size_t size
) {
if (rle) {
size_t compressedSize;
auto compressed = compress(data.get(), size, compressedSize);
put(x, z, layer, std::move(compressed), compressedSize, false);
return;
auto& layer = layers[layerid];
if (layer.compression != compression::Method::NONE) {
data = compression::compress(
data.get(), size, size, layer.compression);
}
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = layers[layer].getOrCreateRegion(regionX, regionZ);
WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, std::move(data), size);
}
@ -139,7 +120,6 @@ static std::unique_ptr<ubyte[]> write_inventories(
return data;
}
/// @brief Store chunk data (voxels and lights) in region (existing or new)
void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
assert(chunk != nullptr);
if (!chunk->flags.lighted) {
@ -157,8 +137,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_VOXELS,
chunk->encode(),
CHUNK_DATA_LEN,
true);
CHUNK_DATA_LEN);
// Writing lights cache
if (doWriteLights && chunk->flags.lighted) {
@ -166,8 +145,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_LIGHTS,
chunk->lightmap.encode(),
LIGHTMAP_DATA_LEN,
true);
LIGHTMAP_DATA_LEN);
}
// Writing block inventories
if (!chunk->inventories.empty()) {
@ -177,8 +155,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_INVENTORIES,
std::move(data),
datasize,
false);
datasize);
}
// Writing entities
if (!entitiesData.empty()) {
@ -190,29 +167,30 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_ENTITIES,
std::move(data),
entitiesData.size(),
false);
entitiesData.size());
}
}
std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
uint32_t size;
auto* data = layers[REGION_LAYER_VOXELS].getData(x, z, size);
auto& layer = layers[REGION_LAYER_VOXELS];
auto* data = layer.getData(x, z, size);
if (data == nullptr) {
return nullptr;
}
return decompress(data, size, CHUNK_DATA_LEN);
return compression::decompress(data, size, CHUNK_DATA_LEN, layer.compression);
}
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
uint32_t size;
auto* bytes = layers[REGION_LAYER_LIGHTS].getData(x, z, size);
auto& layer = layers[REGION_LAYER_LIGHTS];
auto* bytes = layer.getData(x, z, size);
if (bytes == nullptr) {
return nullptr;
}
auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
auto data = compression::decompress(
bytes, size, LIGHTMAP_DATA_LEN, layer.compression
);
return Lightmap::decode(data.get());
}
@ -254,10 +232,11 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) {
}
void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
if (layers[REGION_LAYER_VOXELS].getRegion(x, z)) {
auto& layer = layers[REGION_LAYER_VOXELS];
if (layer.getRegion(x, z)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto regfile = layers[REGION_LAYER_VOXELS].getRegFile({x, z});
auto regfile = layer.getRegFile({x, z});
if (regfile == nullptr) {
throw std::runtime_error("could not open region file");
}
@ -271,14 +250,15 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
if (data == nullptr) {
continue;
}
data = decompress(data.get(), length, CHUNK_DATA_LEN);
data = compression::decompress(
data.get(), length, CHUNK_DATA_LEN, layer.compression
);
if (func(data.get())) {
put(gx,
gz,
REGION_LAYER_VOXELS,
std::move(data),
CHUNK_DATA_LEN,
true);
CHUNK_DATA_LEN);
}
}
}

36
src/files/WorldRegions.hpp
View File

@ -8,11 +8,12 @@
#include <mutex>
#include <unordered_map>
#include "data/dynamic_fwd.hpp"
#include "typedefs.hpp"
#include "data/dynamic_fwd.hpp"
#include "util/BufferPool.hpp"
#include "voxels/Chunk.hpp"
#include "maths/voxmaths.hpp"
#include "coders/compression.hpp"
#include "files.hpp"
#define GLM_ENABLE_EXPERIMENTAL
@ -126,6 +127,8 @@ struct RegionsLayer {
/// @brief Regions layer folder
fs::path folder;
compression::Method compression = compression::Method::NONE;
/// @brief In-memory regions data
regionsmap regions;
@ -139,7 +142,7 @@ struct RegionsLayer {
std::mutex regFilesMutex;
std::condition_variable regFilesCv;
regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
[[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
[[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord);
regfile_ptr createRegFile(glm::ivec2 coord);
void closeRegFile(glm::ivec2 coord);
@ -178,26 +181,6 @@ class WorldRegions {
fs::path directory;
RegionsLayer layers[4] {};
util::BufferPool<ubyte> bufferPool {
std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2};
/// @brief Compress buffer with extrle
/// @param src source buffer
/// @param srclen length of the source buffer
/// @param len (out argument) length of result buffer
/// @return compressed bytes array
std::unique_ptr<ubyte[]> compress(
const ubyte* src, size_t srclen, size_t& len
);
/// @brief Decompress buffer with extrle
/// @param src compressed buffer
/// @param srclen length of compressed buffer
/// @param dstlen max expected length of source buffer
/// @return decompressed bytes array
std::unique_ptr<ubyte[]> decompress(
const ubyte* src, size_t srclen, size_t dstlen
);
public:
bool generatorTestMode = false;
bool doWriteLights = true;
@ -215,19 +198,22 @@ public:
/// @param layer regions layer
/// @param data target data
/// @param size data size
/// @param rle compress with ext-RLE
void put(
int x,
int z,
int layer,
std::unique_ptr<ubyte[]> data,
size_t size,
bool rle
size_t size
);
std::unique_ptr<ubyte[]> getVoxels(int x, int z);
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> getLights(int x, int z);
chunk_inventories_map fetchInventories(int x, int z);
dynamic::Map_sptr fetchEntities(int x, int z);
void processRegionVoxels(int x, int z, const regionproc& func);

4
src/util/BufferPool.hpp
View File

@ -35,5 +35,9 @@ namespace util {
freeBuffers.push(ptr);
});
}
size_t getBufferSize() const {
return bufferSize;
}
};
}

7
src/voxels/ChunksStorage.cpp
View File

@ -59,8 +59,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
auto chunk = std::make_shared<Chunk>(x, z);
store(chunk);
auto data = regions.getVoxels(chunk->x, chunk->z);
if (data) {
if (auto data = regions.getVoxels(chunk->x, chunk->z)) {
chunk->decode(data.get());
auto invs = regions.fetchInventories(chunk->x, chunk->z);
@ -77,9 +76,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
}
verifyLoadedChunk(level->content->getIndices(), chunk.get());
}
auto lights = regions.getLights(chunk->x, chunk->z);
if (lights) {
if (auto lights = regions.getLights(chunk->x, chunk->z)) {
chunk->lightmap.set(lights.get());
chunk->flags.loadedLights = true;
}