From cc58ea29bc823b0faac2c144a486801a6254576c Mon Sep 17 00:00:00 2001
From: MihailRis <mexaruc@gmail.com>
Date: Mon, 15 Apr 2024 19:44:21 +0300
Subject: [PATCH] world converter threading

---
 src/assets/AssetsLoader.cpp            |  4 +-
 src/files/WorldConverter.cpp           | 98 ++++++++++++++++++++++----
 src/files/WorldConverter.h             | 44 +++++-------
 src/files/WorldRegions.cpp             |  2 +-
 src/frontend/menu/menu.cpp             | 12 ++--
 src/frontend/menu/menu_settings.cpp    |  3 +-
 src/graphics/render/ChunksRenderer.cpp |  2 +-
 src/graphics/render/ChunksRenderer.h   |  2 +-
 src/interfaces/Task.h                  |  2 +-
 src/util/ThreadPool.h                  | 16 ++---
 10 files changed, 122 insertions(+), 63 deletions(-)

diff --git a/src/assets/AssetsLoader.cpp b/src/assets/AssetsLoader.cpp
index e1b550c5..85dc5641 100644
--- a/src/assets/AssetsLoader.cpp
+++ b/src/assets/AssetsLoader.cpp
@@ -208,7 +208,7 @@ const ResPaths* AssetsLoader::getPaths() const {
     return paths;
 }
 
-class LoaderWorker : public util::Worker<std::shared_ptr<aloader_entry>, assetload::postfunc> {
+class LoaderWorker : public util::Worker<aloader_entry, assetload::postfunc> {
     AssetsLoader* loader;
 public:
     LoaderWorker(AssetsLoader* loader) : loader(loader) {
@@ -222,7 +222,7 @@ public:
 
 std::shared_ptr<Task> AssetsLoader::startTask(runnable onDone) {
     auto pool = std::make_shared<
-        util::ThreadPool<std::shared_ptr<aloader_entry>, assetload::postfunc>
+        util::ThreadPool<aloader_entry, assetload::postfunc>
     >(
         "assets-loader-pool", 
         [=](){return std::make_shared<LoaderWorker>(this);},
diff --git a/src/files/WorldConverter.cpp b/src/files/WorldConverter.cpp
index 045faf4e..8c81b1d8 100644
--- a/src/files/WorldConverter.cpp
+++ b/src/files/WorldConverter.cpp
@@ -11,16 +11,29 @@
 #include "../content/ContentLUT.h"
 #include "../objects/Player.h"
 #include "../debug/Logger.h"
+#include "../util/ThreadPool.h"
 
 namespace fs = std::filesystem;
 
 static debug::Logger logger("world-converter");
 
+class ConverterWorker : public util::Worker<convert_task, int> {
+    std::shared_ptr<WorldConverter> converter;
+public:
+    ConverterWorker(std::shared_ptr<WorldConverter> converter) 
+    : converter(converter) {}
+
+    int operator()(const std::shared_ptr<convert_task>& task) override {
+        converter->convert(*task);
+        return 0;
+    }
+};
+
 WorldConverter::WorldConverter(
     fs::path folder, 
     const Content* content, 
     std::shared_ptr<ContentLUT> lut
-) : wfile(std::make_unique<WorldFiles>(folder, DebugSettings {})), 
+) : wfile(std::make_unique<WorldFiles>(folder)), 
     lut(lut), 
     content(content) 
 {
@@ -38,7 +51,40 @@ WorldConverter::WorldConverter(
 WorldConverter::~WorldConverter() {
 }
 
-void WorldConverter::convertRegion(fs::path file) {
+std::shared_ptr<Task> WorldConverter::startTask(
+    fs::path folder, 
+    const Content* content, 
+    std::shared_ptr<ContentLUT> lut,
+    runnable onDone,
+    bool multithreading
+) {
+    auto converter = std::make_shared<WorldConverter>(folder, content, lut);
+    if (!multithreading) {
+        converter->setOnComplete([=]() {
+            converter->write();
+            onDone();
+        });
+        return converter;
+    }
+    auto pool = std::make_shared<util::ThreadPool<convert_task, int>>(
+        "converter-pool",
+        [=](){return std::make_shared<ConverterWorker>(converter);},
+        [=](int& _) {}
+    );
+    while (!converter->tasks.empty()) {
+        const convert_task& task = converter->tasks.front();
+        auto ptr = std::make_shared<convert_task>(task);
+        pool->enqueueJob(ptr);
+        converter->tasks.pop();
+    }
+    pool->setOnComplete([=]() {
+        converter->write();
+        onDone();
+    });
+    return pool;
+}
+
+void WorldConverter::convertRegion(fs::path file) const {
     int x, z;
     std::string name = file.stem().string();
     if (!WorldRegions::parseRegionFilename(name, x, z)) {
@@ -54,13 +100,27 @@ void WorldConverter::convertRegion(fs::path file) {
     });
 }
 
-void WorldConverter::convertPlayer(fs::path file) {
+void WorldConverter::convertPlayer(fs::path file) const {
     logger.info() << "converting player " << file.u8string();
     auto map = files::read_json(file);
     Player::convert(map.get(), lut.get());
     files::write_json(file, map.get());
 }
 
+void WorldConverter::convert(convert_task task) const {
+    if (!fs::is_regular_file(task.file))
+        return;
+
+    switch (task.type) {
+        case convert_task_type::region:
+            convertRegion(task.file);
+            break;
+        case convert_task_type::player:
+            convertPlayer(task.file);
+            break;
+    }
+}
+
 void WorldConverter::convertNext() {
     if (tasks.empty()) {
         throw std::runtime_error("no more regions to convert");
@@ -69,18 +129,28 @@ void WorldConverter::convertNext() {
     tasks.pop();
     tasksDone++;
 
-    if (!fs::is_regular_file(task.file))
-        return;
-    switch (task.type) {
-        case convert_task_type::region:
-            convertRegion(task.file);
-            break;
-        case convert_task_type::player:
-            convertPlayer(task.file);
-            break;
+    convert(task);
+}
+
+void WorldConverter::setOnComplete(runnable callback) {
+    this->onComplete = callback;
+}
+
+void WorldConverter::update() {
+    convertNext();
+    if (onComplete && tasks.empty()) {
+        onComplete();
     }
 }
 
+void WorldConverter::terminate() {
+    tasks = {};
+}
+
+bool WorldConverter::isActive() const {
+    return !tasks.empty();
+}
+
 void WorldConverter::write() {
     logger.info() << "writing world";
     wfile->write(nullptr, content);
@@ -92,8 +162,8 @@ void WorldConverter::waitForEnd() {
     }
 }
 
-uint WorldConverter::getWorkRemaining() const {
-    return tasks.size();
+uint WorldConverter::getWorkTotal() const {
+    return tasks.size() + tasksDone;
 }
 
 uint WorldConverter::getWorkDone() const {
diff --git a/src/files/WorldConverter.h b/src/files/WorldConverter.h
index 481d323d..9bc3f52a 100644
--- a/src/files/WorldConverter.h
+++ b/src/files/WorldConverter.h
@@ -32,41 +32,35 @@ class WorldConverter : public Task {
     runnable onComplete;
     uint tasksDone = 0;
 
-    void convertPlayer(fs::path file);
-    void convertRegion(fs::path file);
+    void convertPlayer(fs::path file) const;
+    void convertRegion(fs::path file) const;
 public:
     WorldConverter(
-        fs::path folder, const Content* content, 
+        fs::path folder, 
+        const Content* content, 
         std::shared_ptr<ContentLUT> lut
     );
     ~WorldConverter();
 
+    void convert(convert_task task) const;
     void convertNext();
-
-    void setOnComplete(runnable callback) {
-        this->onComplete = callback;
-    }
-
-    void update() override {
-        convertNext();
-        if (onComplete && tasks.empty()) {
-            onComplete();
-        }
-    }
-
-    void terminate() override {
-        tasks = {};
-    }
-
-    bool isActive() const override {
-        return !tasks.empty();
-    }
-
-    void waitForEnd() override;
+    void setOnComplete(runnable callback);
     void write();
 
-    uint getWorkRemaining() const override;
+    void update() override;
+    void terminate() override;
+    bool isActive() const override;
+    void waitForEnd() override;
+    uint getWorkTotal() const override;
     uint getWorkDone() const override;
+
+    static std::shared_ptr<Task> startTask(
+        fs::path folder, 
+        const Content* content, 
+        std::shared_ptr<ContentLUT> lut,
+        runnable onDone,
+        bool multithreading
+    );
 };
 
 #endif // FILES_WORLD_CONVERTER_H_
diff --git a/src/files/WorldRegions.cpp b/src/files/WorldRegions.cpp
index 9d139de7..9be46960 100644
--- a/src/files/WorldRegions.cpp
+++ b/src/files/WorldRegions.cpp
@@ -207,7 +207,7 @@ ubyte* WorldRegions::getData(
 std::shared_ptr<regfile> WorldRegions::useRegFile(glm::ivec3 coord) {
     auto* file = openRegFiles[coord].get();
     file->inUse = true;
-    return std::shared_ptr<regfile>(openRegFiles[coord].get(), [this](regfile* ptr) {
+    return std::shared_ptr<regfile>(file, [this](regfile* ptr) {
         ptr->inUse = false;
         regFilesCv.notify_one();
     });
diff --git a/src/frontend/menu/menu.cpp b/src/frontend/menu/menu.cpp
index d06c677e..889a3739 100644
--- a/src/frontend/menu/menu.cpp
+++ b/src/frontend/menu/menu.cpp
@@ -116,7 +116,7 @@ void show_process_panel(Engine* engine, std::shared_ptr<Task> task, std::wstring
     auto label = std::make_shared<Label>(L"0%");
     panel->add(label);
 
-    uint initialWork = task->getWorkRemaining();
+    uint initialWork = task->getWorkTotal();
 
     panel->listenInterval(0.01f, [=]() {
         task->update();
@@ -134,25 +134,21 @@ void show_process_panel(Engine* engine, std::shared_ptr<Task> task, std::wstring
     menu->setPage("process", false);
 }
 
-std::shared_ptr<WorldConverter> create_converter(
+std::shared_ptr<Task> create_converter(
     Engine* engine,
     fs::path folder, 
     const Content* content, 
     std::shared_ptr<ContentLUT> lut, 
     runnable postRunnable)
 {
-    auto converter = std::make_shared<WorldConverter>(folder, content, lut);
-    converter->setOnComplete([=](){
-        converter->write();
-
+    return WorldConverter::startTask(folder, content, lut, [=](){
         auto menu = engine->getGUI()->getMenu();
         menu->reset();
         menu->setPage("main", false);
         engine->getGUI()->postRunnable([=]() {
             postRunnable();
         });
-    });
-    return converter;
+    }, true);
 }
 
 void show_convert_request(
diff --git a/src/frontend/menu/menu_settings.cpp b/src/frontend/menu/menu_settings.cpp
index 3ed2ec73..ec793baf 100644
--- a/src/frontend/menu/menu_settings.cpp
+++ b/src/frontend/menu/menu_settings.cpp
@@ -34,7 +34,7 @@ static void create_controls_panel(Engine* engine) {
     }
 
     auto scrollPanel = std::dynamic_pointer_cast<Panel>(guiutil::create(
-        "<panel size='400,200' padding='2' interval='1' max-length='400' color='#0000004C'>"
+        "<panel size='380,200' padding='2' interval='1' max-length='400' color='#0000004C'>"
         "</panel>"
     ));
     for (auto& entry : Events::bindings){
@@ -55,7 +55,6 @@ static void create_controls_panel(Engine* engine) {
 }
 
 void menus::create_settings_panel(Engine* engine) {
-    //create_audio_settings_panel(engine);
     create_controls_panel(engine);
 
     auto menu = engine->getGUI()->getMenu();
diff --git a/src/graphics/render/ChunksRenderer.cpp b/src/graphics/render/ChunksRenderer.cpp
index 8b20414c..b91f4d65 100644
--- a/src/graphics/render/ChunksRenderer.cpp
+++ b/src/graphics/render/ChunksRenderer.cpp
@@ -14,7 +14,7 @@ static debug::Logger logger("chunks-render");
 
 const uint RENDERER_CAPACITY = 9 * 6 * 6 * 3000;
 
-class RendererWorker : public util::Worker<std::shared_ptr<Chunk>, RendererResult> {
+class RendererWorker : public util::Worker<Chunk, RendererResult> {
     Level* level;
     BlocksRenderer renderer;
 public:
diff --git a/src/graphics/render/ChunksRenderer.h b/src/graphics/render/ChunksRenderer.h
index ca100218..d64083bb 100644
--- a/src/graphics/render/ChunksRenderer.h
+++ b/src/graphics/render/ChunksRenderer.h
@@ -29,7 +29,7 @@ class ChunksRenderer {
     std::unordered_map<glm::ivec2, std::shared_ptr<Mesh>> meshes;
     std::unordered_map<glm::ivec2, bool> inwork;
 
-    util::ThreadPool<std::shared_ptr<Chunk>, RendererResult> threadPool;
+    util::ThreadPool<Chunk, RendererResult> threadPool;
 public:
     ChunksRenderer(
         Level* level, 
diff --git a/src/interfaces/Task.h b/src/interfaces/Task.h
index 0b3a2597..df8b3499 100644
--- a/src/interfaces/Task.h
+++ b/src/interfaces/Task.h
@@ -10,7 +10,7 @@ public:
     virtual ~Task() {}
 
     virtual bool isActive() const = 0;
-    virtual uint getWorkRemaining() const = 0;
+    virtual uint getWorkTotal() const = 0;
     virtual uint getWorkDone() const = 0;
     virtual void update() = 0;
     virtual void waitForEnd() = 0;
diff --git a/src/util/ThreadPool.h b/src/util/ThreadPool.h
index e992f139..7480a0a8 100644
--- a/src/util/ThreadPool.h
+++ b/src/util/ThreadPool.h
@@ -17,7 +17,7 @@ namespace util {
     
 template<class J, class T>
 struct ThreadPoolResult {
-    J job;
+    std::shared_ptr<J> job;
     std::condition_variable& variable;
     int workerIndex;
     bool& locked;
@@ -29,13 +29,13 @@ class Worker {
 public:
     Worker() {}
     virtual ~Worker() {}
-    virtual R operator()(const T&) = 0;
+    virtual R operator()(const std::shared_ptr<T>&) = 0;
 };
 
 template<class T, class R>
 class ThreadPool : public Task {
     debug::Logger logger;
-    std::queue<T> jobs;
+    std::queue<std::shared_ptr<T>> jobs;
     std::queue<ThreadPoolResult<T, R>> results;
     std::mutex resultsMutex;
     std::vector<std::thread> threads;
@@ -43,7 +43,7 @@ class ThreadPool : public Task {
     std::mutex jobsMutex;
     std::vector<std::unique_lock<std::mutex>> workersBlocked;
     consumer<R&> resultConsumer;
-    consumer<T&> onJobFailed = nullptr;
+    consumer<std::shared_ptr<T>&> onJobFailed = nullptr;
     runnable onComplete = nullptr;
     std::atomic<int> busyWorkers = 0;
     std::atomic<uint> jobsDone = 0;
@@ -57,7 +57,7 @@ class ThreadPool : public Task {
         std::mutex mutex;
         bool locked = false;
         while (working) {
-            T job;
+            std::shared_ptr<T> job;
             {
                 std::unique_lock<std::mutex> lock(jobsMutex);
                 jobsMutexCondition.wait(lock, [this] {
@@ -199,7 +199,7 @@ public:
         }
     }
 
-    void enqueueJob(T job) {
+    void enqueueJob(std::shared_ptr<T> job) {
         {
             std::lock_guard<std::mutex> lock(jobsMutex);
             jobs.push(job);
@@ -228,8 +228,8 @@ public:
         this->onComplete = callback;
     }
 
-    uint getWorkRemaining() const override {
-        return jobs.size();
+    uint getWorkTotal() const override {
+        return jobs.size()+jobsDone+busyWorkers;
     }
 
     uint getWorkDone() const override {