PulseStudio/VoxelEngine
PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

thread pool update + refactor

Browse Source
This commit is contained in:
MihailRis 2024-04-11 15:48:27 +03:00
parent 1549a02731
commit 90d0b54a69
17 changed files with 271 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
src/assets/AssetsLoader.cpp
View File

@ -6,6 +6,8 @@
#include <iostream>
#include <memory>
#include "../util/ThreadPool.h"
#include "../constants.h"
#include "../data/dynamic.h"
#include "../debug/Logger.h"
@ -21,37 +23,42 @@ static debug::Logger logger("assets-loader");
AssetsLoader::AssetsLoader(Assets* assets, const ResPaths* paths)
: assets(assets), paths(paths)
{
addLoader(AssetType::shader, assetload::shader);
addLoader(AssetType::texture, assetload::texture);
addLoader(AssetType::font, assetload::font);
addLoader(AssetType::atlas, assetload::atlas);
addLoader(AssetType::shader, assetload::shader);
addLoader(AssetType::texture, assetload::texture);
addLoader(AssetType::font, assetload::font);
addLoader(AssetType::atlas, assetload::atlas);
addLoader(AssetType::layout, assetload::layout);
addLoader(AssetType::sound, assetload::sound);
}
void AssetsLoader::addLoader(AssetType tag, aloader_func func) {
loaders[tag] = func;
loaders[tag] = func;
}
void AssetsLoader::add(AssetType tag, const std::string filename, const std::string alias, std::shared_ptr<AssetCfg> settings) {
entries.push(aloader_entry{tag, filename, alias, settings});
entries.push(aloader_entry{tag, filename, alias, settings});
}
bool AssetsLoader::hasNext() const {
return !entries.empty();
return !entries.empty();
}
aloader_func AssetsLoader::getLoader(AssetType tag) {
auto found = loaders.find(tag);
if (found == loaders.end()) {
throw std::runtime_error(
"unknown asset tag "+std::to_string(static_cast<int>(tag))
);
}
return found->second;
}
bool AssetsLoader::loadNext() {
const aloader_entry& entry = entries.front();
logger.info() << "loading " << entry.filename << " as " << entry.alias;
auto found = loaders.find(entry.tag);
if (found == loaders.end()) {
logger.error() << "unknown asset tag " << static_cast<int>(entry.tag);
return false;
}
aloader_func loader = found->second;
const aloader_entry& entry = entries.front();
logger.info() << "loading " << entry.filename << " as " << entry.alias;
try {
auto postfunc = loader(*this, assets, paths, entry.filename, entry.alias, entry.config);
aloader_func loader = getLoader(entry.tag);
auto postfunc = loader(this, paths, entry.filename, entry.alias, entry.config);
postfunc(assets);
entries.pop();
return true;
@ -198,5 +205,37 @@ void AssetsLoader::addDefaults(AssetsLoader& loader, const Content* content) {
}
const ResPaths* AssetsLoader::getPaths() const {
return paths;
return paths;
}
class LoaderWorker : public util::Worker<std::shared_ptr<aloader_entry>, assetload::postfunc> {
AssetsLoader* loader;
public:
LoaderWorker(AssetsLoader* loader) : loader(loader) {
}
assetload::postfunc operator()(const std::shared_ptr<aloader_entry>& entry) override {
aloader_func loadfunc = loader->getLoader(entry->tag);
return loadfunc(loader, loader->getPaths(), entry->filename, entry->alias, entry->config);
}
};
std::shared_ptr<Task> AssetsLoader::startTask(runnable onDone) {
auto pool = std::make_shared<
util::ThreadPool<std::shared_ptr<aloader_entry>, assetload::postfunc>
>(
"assets-loader-pool",
[=](){return std::make_shared<LoaderWorker>(this);},
[=](assetload::postfunc& func) {
func(assets);
}
);
pool->setOnComplete(onDone);
while (!entries.empty()) {
const aloader_entry& entry = entries.front();
auto ptr = std::make_shared<aloader_entry>(entry);
pool->enqueueJob(ptr);
entries.pop();
}
return pool;
}

8
src/assets/AssetsLoader.h
View File

@ -2,6 +2,8 @@
#define ASSETS_ASSETS_LOADER_H
#include "Assets.h"
#include "../interfaces/Task.h"
#include "../delegates.h"
#include <string>
#include <memory>
@ -45,8 +47,7 @@ struct SoundCfg : AssetCfg {
};
using aloader_func = std::function<assetload::postfunc(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths*,
const std::string&,
const std::string&,
@ -96,7 +97,10 @@ public:
/// @param content engine content
static void addDefaults(AssetsLoader& loader, const Content* content);
std::shared_ptr<Task> startTask(runnable onDone);
const ResPaths* getPaths() const;
aloader_func getLoader(AssetType tag);
};
#endif // ASSETS_ASSETS_LOADER_H

30
src/assets/assetload_funcs.cpp
View File

@ -8,7 +8,7 @@
#include "../audio/audio.h"
#include "../files/files.h"
#include "../files/engine_paths.h"
#include "../coders/png.h"
#include "../coders/imageio.h"
#include "../coders/json.h"
#include "../coders/GLSLExtension.h"
#include "../graphics/core/Shader.h"
@ -30,15 +30,14 @@ static bool animation(
);
assetload::postfunc assetload::texture(
AssetsLoader&,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
std::shared_ptr<AssetCfg>
) {
std::shared_ptr<ImageData> image (
png::load_image(paths->find(filename+".png").u8string())
imageio::read(paths->find(filename+".png").u8string()).release()
);
return [name, image](auto assets) {
assets->store(Texture::from(image.get()), name);
@ -46,8 +45,7 @@ assetload::postfunc assetload::texture(
}
assetload::postfunc assetload::shader(
AssetsLoader&,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
@ -72,15 +70,12 @@ assetload::postfunc assetload::shader(
}
static bool appendAtlas(AtlasBuilder& atlas, const fs::path& file) {
// png is only supported format
if (file.extension() != ".png")
return false;
std::string name = file.stem().string();
// skip duplicates
if (atlas.has(name)) {
return false;
}
std::unique_ptr<ImageData> image(png::load_image(file.string()));
auto image = imageio::read(file.string());
image->fixAlphaColor();
atlas.add(name, image.release());
@ -88,8 +83,7 @@ static bool appendAtlas(AtlasBuilder& atlas, const fs::path& file) {
}
assetload::postfunc assetload::atlas(
AssetsLoader&,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string directory,
const std::string name,
@ -114,8 +108,7 @@ assetload::postfunc assetload::atlas(
}
assetload::postfunc assetload::font(
AssetsLoader&,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
@ -125,8 +118,7 @@ assetload::postfunc assetload::font(
for (size_t i = 0; i <= 4; i++) {
std::string name = filename + "_" + std::to_string(i) + ".png";
name = paths->find(name).string();
std::unique_ptr<ImageData> image (png::load_image(name));
pages->push_back(std::move(image));
pages->push_back(std::move(imageio::read(name)));
}
return [=](auto assets) {
int res = pages->at(0)->getHeight() / 16;
@ -139,8 +131,7 @@ assetload::postfunc assetload::font(
}
assetload::postfunc assetload::layout(
AssetsLoader& loader,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string file,
const std::string name,
@ -159,8 +150,7 @@ assetload::postfunc assetload::layout(
};
}
assetload::postfunc assetload::sound(
AssetsLoader& loader,
Assets* assets,
AssetsLoader*,
const ResPaths* paths,
const std::string file,
const std::string name,

18
src/assets/assetload_funcs.h
View File

@ -15,40 +15,35 @@ struct AssetCfg;
/// @brief see AssetsLoader.h: aloader_func
namespace assetload {
postfunc texture(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
std::shared_ptr<AssetCfg> settings
);
postfunc shader(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
std::shared_ptr<AssetCfg> settings
);
postfunc atlas(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string directory,
const std::string name,
std::shared_ptr<AssetCfg> settings
);
postfunc font(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string filename,
const std::string name,
std::shared_ptr<AssetCfg> settings
);
postfunc layout(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string file,
const std::string name,
@ -56,8 +51,7 @@ namespace assetload {
);
postfunc sound(
AssetsLoader&,
Assets*,
AssetsLoader*,
const ResPaths* paths,
const std::string file,
const std::string name,

49
src/coders/imageio.cpp Normal file
View File

@ -0,0 +1,49 @@
#include "imageio.h"
#include "png.h"
#include "../graphics/core/ImageData.h"
#include <filesystem>
#include <functional>
#include <unordered_map>
namespace fs = std::filesystem;
using image_reader = std::function<ImageData*(const std::string&)>;
using image_writer = std::function<void(const std::string&, const ImageData*)>;
static std::unordered_map<std::string, image_reader> readers {
{".png", png::load_image},
};
static std::unordered_map<std::string, image_writer> writers {
{".png", png::write_image},
};
bool imageio::is_read_supported(const std::string& extension) {
return readers.find(extension) != readers.end();
}
bool imageio::is_write_supported(const std::string& extension) {
return writers.find(extension) != writers.end();
}
inline std::string extensionOf(const std::string& filename) {
return fs::u8path(filename).extension().u8string();
}
std::unique_ptr<ImageData> imageio::read(const std::string& filename) {
auto found = readers.find(extensionOf(filename));
if (found == readers.end()) {
throw std::runtime_error("file format is not supported (read): "+filename);
}
return std::unique_ptr<ImageData>(found->second(filename));
}
void imageio::write(const std::string& filename, const ImageData* image) {
auto found = writers.find(extensionOf(filename));
if (found == writers.end()) {
throw std::runtime_error("file format is not supported (write): "+filename);
}
return found->second(filename, image);
}

19
src/coders/imageio.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef CODERS_IMAGEIO_H_
#define CODERS_IMAGEIO_H_
#include <string>
#include <memory>
class ImageData;
namespace imageio {
inline const std::string PNG = ".png";
bool is_read_supported(const std::string& extension);
bool is_write_supported(const std::string& extension);
std::unique_ptr<ImageData> read(const std::string& filename);
void write(const std::string& filename, const ImageData* image);
}
#endif // CODERS_IMAGEIO_H_

6
src/coders/png.cpp
View File

@ -340,7 +340,7 @@ ImageData* _png_load(const char* file){
}
#endif
ImageData* png::load_image(std::string filename) {
ImageData* png::load_image(const std::string& filename) {
ImageData* image (_png_load(filename.c_str()));
if (image == nullptr) {
throw std::runtime_error("could not load image "+filename);
@ -348,13 +348,13 @@ ImageData* png::load_image(std::string filename) {
return image;
}
Texture* png::load_texture(std::string filename) {
Texture* png::load_texture(const std::string& filename) {
std::unique_ptr<ImageData> image (load_image(filename));
auto texture = Texture::from(image.get());
texture->setNearestFilter();
return texture;
}
void png::write_image(std::string filename, const ImageData* image) {
void png::write_image(const std::string& filename, const ImageData* image) {
_png_write(filename.c_str(), image->getWidth(), image->getHeight(), (const ubyte*)image->getData(), image->getFormat() == ImageFormat::rgba8888);
}

6
src/coders/png.h
View File

@ -8,9 +8,9 @@ class Texture;
class ImageData;
namespace png {
extern ImageData* load_image(std::string filename);
extern void write_image(std::string filename, const ImageData* image);
extern Texture* load_texture(std::string filename);
extern ImageData* load_image(const std::string& filename);
extern void write_image(const std::string& filename, const ImageData* image);
extern Texture* load_texture(const std::string& filename);
}
#endif /* CODERS_PNG_H_ */

40
src/engine.cpp
View File

@ -6,7 +6,7 @@
#include "audio/audio.h"
#include "coders/GLSLExtension.h"
#include "coders/json.h"
#include "coders/png.h"
#include "coders/imageio.h"
#include "content/ContentLoader.h"
#include "core_defs.h"
#include "files/files.h"
@ -70,7 +70,6 @@ Engine::Engine(EngineSettings& settings, EnginePaths* paths)
auto resdir = paths->getResources();
logger.info() << "loading assets";
std::vector<fs::path> roots {resdir};
resPaths = std::make_unique<ResPaths>(resdir, roots);
try {
@ -115,17 +114,21 @@ void Engine::updateTimers() {
void Engine::updateHotkeys() {
if (Events::jpressed(keycode::F2)) {
std::unique_ptr<ImageData> image(Window::takeScreenshot());
image->flipY();
fs::path filename = paths->getScreenshotFile("png");
png::write_image(filename.string(), image.get());
std::cout << "saved screenshot as " << filename << std::endl;
saveScreenshot();
}
if (Events::jpressed(keycode::F11)) {
Window::toggleFullscreen();
}
}
void Engine::saveScreenshot() {
std::unique_ptr<ImageData> image(Window::takeScreenshot());
image->flipY();
fs::path filename = paths->getScreenshotFile("png");
imageio::write(filename.string(), image.get());
logger.info() << "saved screenshot as "+filename.u8string();
}
void Engine::mainloop() {
logger.info() << "starting menu screen";
setScreen(std::make_shared<MenuScreen>(this));
@ -203,10 +206,17 @@ void Engine::loadAssets() {
auto new_assets = std::make_unique<Assets>();
AssetsLoader loader(new_assets.get(), resPaths.get());
AssetsLoader::addDefaults(loader, content.get());
while (loader.hasNext()) {
if (!loader.loadNext()) {
new_assets.reset();
throw std::runtime_error("could not to load assets");
bool threading = false;
if (threading) {
auto task = loader.startTask([=](){});
task->waitForEnd();
} else {
while (loader.hasNext()) {
if (!loader.loadNext()) {
new_assets.reset();
throw std::runtime_error("could not to load assets");
}
}
}
if (assets) {
@ -216,7 +226,6 @@ void Engine::loadAssets() {
}
}
// TODO: refactor this
void Engine::loadContent() {
auto resdir = paths->getResources();
ContentBuilder contentBuilder;
@ -263,12 +272,7 @@ void Engine::loadWorldContent(const fs::path& folder) {
}
void Engine::loadAllPacks() {
PacksManager manager;
manager.setSources({
paths->getWorldFolder()/fs::path("content"),
paths->getUserfiles()/fs::path("content"),
paths->getResources()/fs::path("content")
});
PacksManager manager = createPacksManager(paths->getWorldFolder());
manager.scan();
auto allnames = manager.getAllNames();
contentPacks = manager.getAll(manager.assembly(allnames));

2
src/engine.h
View File

@ -123,6 +123,8 @@ public:
/// @brief Enqueue function call to the end of current frame in draw thread
void postRunnable(runnable callback);
void saveScreenshot();
PacksManager createPacksManager(const fs::path& worldFolder);
SettingsHandler& getSettingsHandler();

6
src/files/WorldConverter.cpp
View File

@ -92,6 +92,12 @@ void WorldConverter::write() {
wfile->write(nullptr, content);
}
void WorldConverter::waitForEnd() {
while (isActive()) {
update();
}
}
uint WorldConverter::getWorkRemaining() const {
return tasks.size();
}

5
src/files/WorldConverter.h
View File

@ -58,6 +58,11 @@ public:
tasks = {};
}
bool isActive() const override {
return !tasks.empty();
}
void waitForEnd() override;
void write();
uint getWorkRemaining() const override;

2
src/frontend/menu/menu.cpp
View File

@ -201,6 +201,7 @@ void menus::open_world(std::string name, Engine* engine, bool confirmConvert) {
try {
engine->loadWorldContent(folder);
} catch (const contentpack_error& error) {
engine->setScreen(std::make_shared<MenuScreen>(engine));
// could not to find or read pack
guiutil::alert(
engine->getGUI(), langs::get(L"error.pack-not-found")+L": "+
@ -208,6 +209,7 @@ void menus::open_world(std::string name, Engine* engine, bool confirmConvert) {
);
return;
} catch (const std::runtime_error& error) {
engine->setScreen(std::make_shared<MenuScreen>(engine));
guiutil::alert(
engine->getGUI(), langs::get(L"Content Error", L"menu")+L": "+
util::str2wstr_utf8(error.what())

13
src/frontend/menu/menu_pause.cpp
View File

@ -1,11 +1,12 @@
#include "menu.h"
#include "menu_commons.h"
#include "../../coders/png.h"
#include "../../coders/imageio.h"
#include "../../content/PacksManager.h"
#include "../../content/ContentLUT.h"
#include "../../engine.h"
#include "../../files/WorldFiles.h"
#include "../../graphics/core/Texture.h"
#include "../../graphics/ui/gui_util.h"
#include "../../logic/LevelController.h"
#include "../../util/stringutil.h"
@ -52,7 +53,8 @@ std::shared_ptr<Container> create_pack_panel(
if (assets->getTexture(icon) == nullptr) {
auto iconfile = pack.folder/fs::path("icon.png");
if (fs::is_regular_file(iconfile)) {
assets->store(png::load_texture(iconfile.string()), icon);
auto image = imageio::read(iconfile.string());
assets->store(Texture::from(image.get()), icon);
} else {
icon = "gui/no_icon";
}
@ -183,12 +185,7 @@ void create_content_panel(Engine* engine, LevelController* controller) {
auto mainPanel = menus::create_page(engine, "content", 550, 0.0f, 5);
auto paths = engine->getPaths();
PacksManager manager;
manager.setSources({
paths->getWorldFolder()/fs::path("content"),
paths->getUserfiles()/fs::path("content"),
paths->getResources()/fs::path("content")
});
PacksManager manager = engine->createPacksManager(paths->getWorldFolder());
manager.scan();
std::vector<ContentPack> scanned = manager.getAll(manager.getAllNames());

1
src/graphics/render/ChunksRenderer.cpp
View File

@ -46,6 +46,7 @@ ChunksRenderer::ChunksRenderer(
})
{
threadPool.setStandaloneResults(false);
threadPool.setStopOnFail(false);
renderer = std::make_unique<BlocksRenderer>(
RENDERER_CAPACITY, level->content, cache, settings
);

2
src/interfaces/Task.h
View File

@ -9,9 +9,11 @@ class Task {
public:
virtual ~Task() {}
virtual bool isActive() const = 0;
virtual uint getWorkRemaining() const = 0;
virtual uint getWorkDone() const = 0;
virtual void update() = 0;
virtual void waitForEnd() = 0;
virtual void terminate() = 0;
};

94
src/util/ThreadPool.h
View File

@ -4,6 +4,7 @@
#include <queue>
#include <atomic>
#include <thread>
#include <chrono>
#include <iostream>
#include <functional>
#include <condition_variable>
@ -14,8 +15,9 @@
namespace util {
template<class T>
template<class J, class T>
struct ThreadPoolResult {
J job;
std::condition_variable& variable;
int workerIndex;
bool& locked;
@ -34,7 +36,7 @@ template<class T, class R>
class ThreadPool : public Task {
debug::Logger logger;
std::queue<T> jobs;
std::queue<ThreadPoolResult<R>> results;
std::queue<ThreadPoolResult<T, R>> results;
std::mutex resultsMutex;
std::vector<std::thread> threads;
std::condition_variable jobsMutexCondition;
@ -46,7 +48,9 @@ class ThreadPool : public Task {
std::atomic<int> busyWorkers = 0;
std::atomic<uint> jobsDone = 0;
bool working = true;
bool failed = false;
bool standaloneResults = true;
bool stopOnFail = true;
void threadLoop(int index, std::shared_ptr<Worker<T, R>> worker) {
std::condition_variable variable;
@ -59,7 +63,7 @@ class ThreadPool : public Task {
jobsMutexCondition.wait(lock, [this] {
return !jobs.empty() || !working;
});
if (!working) {
if (!working || failed) {
break;
}
job = jobs.front();
@ -71,7 +75,7 @@ class ThreadPool : public Task {
R result = (*worker)(job);
{
std::lock_guard<std::mutex> lock(resultsMutex);
results.push(ThreadPoolResult<R> {variable, index, locked, result});
results.push(ThreadPoolResult<T, R> {job, variable, index, locked, result});
if (!standaloneResults) {
locked = true;
}
@ -88,6 +92,10 @@ class ThreadPool : public Task {
if (onJobFailed) {
onJobFailed(job);
}
if (stopOnFail) {
std::lock_guard<std::mutex> lock(jobsMutex);
failed = true;
}
logger.error() << "uncaught exception: " << err.what();
}
jobsDone++;
@ -109,6 +117,10 @@ public:
terminate();
}
bool isActive() const override {
return working;
}
void terminate() override {
if (!working) {
return;
@ -120,7 +132,7 @@ public:
{
std::lock_guard<std::mutex> lock(resultsMutex);
while (!results.empty()) {
ThreadPoolResult<R> entry = results.front();
ThreadPoolResult<T,R> entry = results.front();
results.pop();
if (!standaloneResults) {
entry.locked = false;
@ -136,24 +148,54 @@ public:
}
void update() override {
std::lock_guard<std::mutex> lock(resultsMutex);
while (!results.empty()) {
ThreadPoolResult<R> entry = results.front();
results.pop();
resultConsumer(entry.entry);
if (!standaloneResults) {
entry.locked = false;
entry.variable.notify_all();
}
if (!working) {
return;
}
if (failed) {
throw std::runtime_error("some job failed");
}
if (onComplete && busyWorkers == 0) {
std::lock_guard<std::mutex> lock(jobsMutex);
if (jobs.empty()) {
onComplete();
bool complete = false;
{
std::lock_guard<std::mutex> lock(resultsMutex);
while (!results.empty()) {
ThreadPoolResult<T,R> entry = results.front();
results.pop();
try {
resultConsumer(entry.entry);
} catch (std::exception& err) {
logger.error() << err.what();
if (onJobFailed) {
onJobFailed(entry.job);
}
if (stopOnFail) {
std::lock_guard<std::mutex> lock(jobsMutex);
failed = true;
complete = false;
}
break;
}
if (!standaloneResults) {
entry.locked = false;
entry.variable.notify_all();
}
}
if (onComplete && busyWorkers == 0) {
std::lock_guard<std::mutex> lock(jobsMutex);
if (jobs.empty()) {
onComplete();
complete = true;
}
}
}
if (failed) {
throw std::runtime_error("some job failed");
}
if (complete) {
terminate();
}
}
@ -170,6 +212,10 @@ public:
standaloneResults = flag;
}
void setStopOnFail(bool flag) {
stopOnFail = flag;
}
/// @brief onJobFailed called on exception thrown in worker thread.
/// Use engine.postRunnable when calling terminate()
void setOnJobFailed(consumer<T&> callback) {
@ -189,6 +235,14 @@ public:
uint getWorkDone() const override {
return jobsDone;
}
virtual void waitForEnd() override {
using namespace std::chrono_literals;
while (working) {
std::this_thread::sleep_for(2ms);
update();
}
}
};
} // namespace util