Merge pull request #678 from MihailRis/memory-fs

Memory Device

dev/tests/filesystem.lua

@@ -4,6 +4,7 @@ assert(file.exists("config:"))
 debug.log("write text file")
 assert(file.write("config:text.txt", "example, пример"))
 assert(file.exists("config:text.txt"))
+assert(file.isfile("config:text.txt"))
 
 debug.log("read text file")
 assert(file.read("config:text.txt") == "example, пример")
@@ -18,11 +19,23 @@ assert(file.isdir("config:dir"))
 
 debug.log("remove directory")
 file.remove("config:dir")
+assert(not file.isdir("config:dir"))
 
 debug.log("create directories")
 file.mkdirs("config:dir/subdir/other")
 assert(file.isdir("config:dir/subdir/other"))
 
+debug.log("list directory")
+file.write("config:dir/subdir/a.txt", "helloworld")
+file.write("config:dir/subdir/b.txt", "gfgsfhs")
+
+local entries = file.list("config:dir/subdir")
+assert(#entries == 3)
+table.sort(entries)
+assert(entries[1] == "config:dir/subdir/a.txt")
+assert(entries[2] == "config:dir/subdir/b.txt")
+assert(entries[3] == "config:dir/subdir/other")
+
 debug.log("remove tree")
 file.remove_tree("config:dir")
 assert(not file.isdir("config:dir"))

dev/tests/memory_filesystem.lua

@@ -0,0 +1,67 @@
+app.create_memory_device("memtest")
+local tmp = file.create_memory_device()
+
+debug.log("check initial state")
+assert(file.exists("memtest:"))
+assert(file.is_writeable("memtest:"))
+assert(file.is_writeable(tmp..":"))
+
+debug.log("write text file")
+assert(file.write("memtest:text.txt", "example, пример"))
+assert(file.exists("memtest:text.txt"))
+assert(file.isfile("memtest:text.txt"))
+
+debug.log("read text file")
+assert(file.read("memtest:text.txt") == "example, пример")
+
+debug.log("delete file")
+file.remove("memtest:text.txt")
+assert(not file.exists("memtest:text.txt"))
+
+debug.log("create directory")
+file.mkdir("memtest:dir")
+assert(file.isdir("memtest:dir"))
+
+debug.log("remove directory")
+file.remove("memtest:dir")
+assert(not file.isdir("memtest:dir"))
+
+debug.log("create directories")
+file.mkdirs("memtest:dir/subdir/other")
+assert(file.isdir("memtest:dir/subdir/other"))
+
+debug.log("list directory")
+file.write("memtest:dir/subdir/a.txt", "helloworld")
+file.write("memtest:dir/subdir/b.txt", "gfgsfhs")
+
+local entries = file.list("memtest:dir/subdir")
+assert(#entries == 3)
+table.sort(entries)
+assert(entries[1] == "memtest:dir/subdir/a.txt")
+assert(entries[2] == "memtest:dir/subdir/b.txt")
+assert(entries[3] == "memtest:dir/subdir/other")
+
+debug.log("remove tree")
+file.remove_tree("memtest:dir")
+assert(not file.isdir("memtest:dir"))
+
+debug.log("write binary file")
+local bytes = {0xDE, 0xAD, 0xC0, 0xDE}
+file.write_bytes("memtest:binary", bytes)
+assert(file.exists("memtest:binary"))
+
+debug.log("write binary file")
+local bytes = {0xDE, 0xAD, 0xC0, 0xDE}
+file.write_bytes("memtest:binary", bytes)
+assert(file.exists("memtest:binary"))
+
+debug.log("read binary file")
+local rbytes = file.read_bytes("memtest:binary")
+assert(#rbytes == #bytes)
+for i, b in ipairs(bytes) do
+    assert(rbytes[i] == b)
+end
+
+debug.log("delete file")
+file.remove("memtest:binary")
+assert(not file.exists("memtest:binary"))

doc/en/scripting/builtins/libapp.md

@@ -158,3 +158,13 @@ app.get_setting_info(name: str) -> {
 ```
 
 Returns a table with information about a setting. Throws an exception if the setting does not exist.
+
+
+```lua
+app.create_memory_device(
+    -- entry-point name
+    name: str
+)
+```
+
+Creates an in-memory filesystem.

doc/en/scripting/builtins/libfile.md

@@ -139,6 +139,12 @@ file.create_zip(directory: str, output_file: str) --> str
 
 Creates a ZIP archive from the contents of the specified directory.
 
+```lua
+file.create_memory_device() --> str
+```
+
+Creates a memory file system and returns entry point name. Lives until content unload.
+
 ```lua
 file.name(path: str) --> str
 ```

doc/ru/scripting/builtins/libapp.md

@@ -159,3 +159,12 @@ app.get_setting_info(name: str) -> {
 ```
 
 Возвращает таблицу с информацией о настройке. Бросает исключение, если настройки не существует.
+
+```lua
+app.create_memory_device(
+    -- имя точки входа
+    name: str
+)
+```
+
+Создаёт файловую систему в памяти.

doc/ru/scripting/builtins/libfile.md

@@ -139,6 +139,12 @@ file.create_zip(директория: str, выходной_файл: str) --> s
 
 Создаёт ZIP-архив из содержимого указанной директории.
 
+```lua
+file.create_memory_device() --> str
+```
+
+Создаёт файловую систему в памяти, возвращает имя точки входа. Удаляется при выгрузке контента.
+
 ```lua
 file.name(путь: str) --> str
 ```

src/engine/EnginePaths.cpp

@@ -2,6 +2,7 @@
 
 #include "debug/Logger.hpp"
 #include "io/devices/StdfsDevice.hpp"
+#include "io/devices/MemoryDevice.hpp"
 #include "io/devices/ZipFileDevice.hpp"
 #include "maths/util.hpp"
 #include "typedefs.hpp"
@@ -168,6 +169,18 @@ void EnginePaths::unmount(const std::string& name) {
     mounted.erase(found);
 }
 
+std::string EnginePaths::createMemoryDevice() {
+    auto device = std::make_unique<io::MemoryDevice>();
+        std::string name;
+    do {
+        name = std::string("W.") + generate_random_base64<6>();
+    } while (std::find(mounted.begin(), mounted.end(), name) != mounted.end());
+    
+    io::set_device(name, std::move(device));
+    mounted.push_back(name);
+    return name;
+}
+
 std::string EnginePaths::createWriteableDevice(const std::string& name) {
     const auto& found = writeables.find(name);
     if (found != writeables.end()) {

src/engine/EnginePaths.hpp

@@ -58,6 +58,7 @@ public:
     void unmount(const std::string& name);
 
     std::string createWriteableDevice(const std::string& name);
+    std::string createMemoryDevice();
 
     void setEntryPoints(std::vector<PathsRoot> entryPoints);
 

src/io/deflate_ostream.hpp

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <iostream>
 #include <streambuf>
 #include <memory>

src/io/devices/MemoryDevice.cpp

@@ -0,0 +1,220 @@
+#include "MemoryDevice.hpp"
+
+#include "../memory_istream.hpp"
+#include "../memory_ostream.hpp"
+#include "../finalizing_ostream.hpp"
+
+#include <algorithm>
+
+io::MemoryDevice::MemoryDevice() {}
+
+std::filesystem::path io::MemoryDevice::resolve(std::string_view path) {
+    throw std::runtime_error("unable to resolve filesystem path");
+}
+
+std::unique_ptr<std::ostream> io::MemoryDevice::write(std::string_view path) {
+    std::string filePath = std::string(path);
+    return std::make_unique<finalizing_ostream>(
+        std::make_unique<memory_ostream>(),
+        [this, filePath](auto ostream) {
+            auto& memoryStream = dynamic_cast<memory_ostream&>(*ostream);
+            createFile(std::move(filePath), memoryStream.release());
+        }
+    );
+}
+
+std::unique_ptr<std::istream> io::MemoryDevice::read(std::string_view path) {
+    const auto& found = nodes.find(std::string(path));
+    if (found == nodes.end()) {
+        return nullptr;
+    }
+    auto& node = found->second;
+    if (auto file = node.get_if<File>()) {
+        if (file->content != nullptr) {
+            return std::make_unique<memory_view_istream>(file->content);
+        }
+    }
+    return nullptr;
+}
+
+size_t io::MemoryDevice::size(std::string_view path) {
+    const auto& found = nodes.find(std::string(path));
+    if (found == nodes.end()) {
+        return 0;
+    }
+    return std::visit([](auto&& arg) -> size_t {
+        using T = std::decay_t<decltype(arg)>;
+        if constexpr (std::is_same_v<T, File>) {
+            return arg.content.size();
+        } else if constexpr (std::is_same_v<T, Dir>) {
+            return arg.content.size();
+        } else {
+            return 0;
+        }
+    }, found->second.data);
+}
+
+io::file_time_type io::MemoryDevice::lastWriteTime(std::string_view path) {
+    return file_time_type::min();
+}
+
+bool io::MemoryDevice::exists(std::string_view path) {
+    if (path.empty()) {
+        return true;
+    }
+    return nodes.find(std::string(path)) != nodes.end();
+}
+
+bool io::MemoryDevice::isdir(std::string_view path) {
+    if (path.empty()) {
+        return true;
+    }
+    const auto& found = nodes.find(std::string(path));
+    if (found == nodes.end()) {
+        return false;
+    }
+    return found->second.holds_alternative<Dir>();
+}
+
+bool io::MemoryDevice::isfile(std::string_view path)  {
+    const auto& found = nodes.find(std::string(path));
+    if (found == nodes.end()) {
+        return false;
+    }
+    return found->second.holds_alternative<File>();
+}
+
+bool io::MemoryDevice::mkdir(std::string_view path) {
+    return createDir(std::string(path)) != nullptr;
+}
+
+bool io::MemoryDevice::mkdirs(std::string_view path) {
+    io::path dirPath = std::string(path);
+    std::vector<std::string> parts;
+    while (!dirPath.pathPart().empty()) {
+        parts.push_back(dirPath.name());
+        dirPath = dirPath.parent();
+    }
+    for (int i = parts.size() - 1; i >= 0; i--) {
+        dirPath = dirPath / parts[i];
+        createDir(dirPath.string());
+    }
+    return true;
+}
+
+bool io::MemoryDevice::remove(std::string_view path) {
+    std::string pathString = std::string(path);
+    const auto& found = nodes.find(pathString);
+    if (found == nodes.end()) {
+        return false;
+    }
+    if (found->second.holds_alternative<Dir>()) {
+        const auto& dir = found->second.get_if<Dir>();
+        if (!dir->content.empty()) {
+            return false;
+        }
+    }
+    io::path filePath = pathString;
+    io::path parentPath = filePath.parent();
+    auto parentDir = getDir(parentPath.string());
+    if (parentDir) {
+        auto& content = parentDir->content;
+        content.erase(
+            std::remove(content.begin(), content.end(), filePath.name()),
+            content.end()
+        );
+    }
+    nodes.erase(found);
+    return true;
+}
+
+uint64_t io::MemoryDevice::removeAll(std::string_view path) {
+    std::string pathString = std::string(path);
+    const auto& found = nodes.find(pathString);
+    if (found == nodes.end()) {
+        return 0;
+    }
+    io::path filePath = pathString;
+
+    uint64_t count = 0;
+    if (found->second.holds_alternative<Dir>()) {
+        auto dir = found->second.get_if<Dir>();
+        auto files = dir->content;
+        for (const auto& name : files) {
+            io::path subPath = filePath / name;
+            count += removeAll(subPath.string());
+        }
+    }
+    if (remove(pathString)) {
+        count++;
+    }
+    return count;
+}
+
+namespace {
+    struct MemoryPathsGenerator : public io::PathsGenerator {
+        std::vector<std::string> entries;
+        size_t index = 0;
+
+        MemoryPathsGenerator(std::vector<std::string>&& entries)
+            : entries(std::move(entries)) {}
+
+        bool next(io::path& outPath) override {
+            if (index >= entries.size()) {
+                return false;
+            }
+            outPath = entries[index++];
+            return true;
+        }
+    };
+}
+
+std::unique_ptr<io::PathsGenerator> io::MemoryDevice::list(std::string_view path) {
+    auto dir = getDir(path);
+    if (!dir) {
+        return nullptr;
+    }
+    return std::make_unique<MemoryPathsGenerator>(
+        std::vector<std::string>(dir->content)
+    );
+}
+
+io::MemoryDevice::Dir* io::MemoryDevice::createDir(std::string path) {
+    io::path filePath = path;
+    io::path parent = filePath.parent();
+    auto parentDir = getDir(parent.string());
+    if (!parentDir) {
+        return nullptr;
+    }
+    parentDir->content.push_back(filePath.name());
+    auto& node = nodes[std::move(path)];
+    node.data = Dir {};
+    return node.get_if<Dir>();
+}
+
+io::MemoryDevice::Node* io::MemoryDevice::createFile(
+    std::string path, util::Buffer<char>&& content
+) {
+    io::path filePath = path;
+    io::path parent = filePath.parent();
+    auto dir = getDir(parent.string());
+    if (!dir) {
+        return nullptr;
+    }
+    dir->content.push_back(filePath.name());
+    auto& node = nodes[std::move(path)];
+    node.data = File {std::move(content)};
+    return &node;
+}
+
+io::MemoryDevice::Dir* io::MemoryDevice::getDir(std::string_view path) {
+    if (path.empty()) {
+        return &rootDir;
+    }
+    const auto& found = nodes.find(std::string(path));
+    if (found == nodes.end()) {
+        return nullptr;
+    }
+    auto& node = found->second;
+    return node.get_if<Dir>();
+}

src/io/devices/MemoryDevice.hpp

@@ -0,0 +1,71 @@
+#pragma once
+
+#include "Device.hpp"
+#include "util/Buffer.hpp"
+
+#include <vector>
+#include <variant>
+#include <optional>
+#include <unordered_map>
+
+namespace io {
+    /// @brief In-memory filesystem device
+    class MemoryDevice : public Device {
+        enum class NodeType {
+            DIR, FILE
+        };
+
+        struct File {
+            util::Buffer<char> content;
+        };
+
+        struct Dir {
+            std::vector<std::string> content;
+        };
+
+        struct Node {
+            std::variant<Dir, File> data;
+
+            NodeType type() const {
+                return std::visit([](auto&& arg) -> NodeType {
+                    using T = std::decay_t<decltype(arg)>;
+                    if constexpr (std::is_same_v<T, Dir>) return NodeType::DIR;
+                    else if constexpr (std::is_same_v<T, File>) return NodeType::FILE;
+                }, data);
+            }
+
+            template <typename T>
+            bool holds_alternative() const {
+                return std::holds_alternative<T>(data);
+            }
+
+            template <typename T>
+            T* get_if() {
+                return std::get_if<T>(&data);
+            }
+        };
+    public:
+        MemoryDevice();
+
+        std::filesystem::path resolve(std::string_view path) override;
+        std::unique_ptr<std::ostream> write(std::string_view path) override;
+        std::unique_ptr<std::istream> read(std::string_view path) override;
+        size_t size(std::string_view path) override;
+        file_time_type lastWriteTime(std::string_view path) override;
+        bool exists(std::string_view path) override;
+        bool isdir(std::string_view path) override;
+        bool isfile(std::string_view path) override;
+        bool mkdir(std::string_view path) override;
+        bool mkdirs(std::string_view path) override;
+        bool remove(std::string_view path) override;
+        uint64_t removeAll(std::string_view path) override;
+        std::unique_ptr<PathsGenerator> list(std::string_view path) override;
+    private:
+        std::unordered_map<std::string, Node> nodes;
+        Dir rootDir {};
+
+        Node* createFile(std::string path, util::Buffer<char>&& content);
+        Dir* createDir(std::string path);
+        Dir* getDir(std::string_view path);
+    };
+}

src/io/devices/StdfsDevice.hpp

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "Device.hpp"
 
 namespace io {

src/io/devices/ZipFileDevice.cpp

@@ -239,7 +239,7 @@ std::unique_ptr<std::istream> ZipFileDevice::read(std::string_view path) {
 size_t ZipFileDevice::size(std::string_view path) {
     const auto& found = entries.find(std::string(path));
     if (found == entries.end()) {
-        return false;
+        return 0;
     }
     return found->second.uncompressedSize;
 }

src/io/finalizing_ostream.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <memory>
+#include <ostream>
+#include <functional>
+
+class finalizing_ostream final : public std::ostream {
+public:
+    finalizing_ostream(
+        std::unique_ptr<std::ostream> inner,
+        std::function<void(std::unique_ptr<std::ostream>)> on_destruction
+    )
+        : std::ostream(inner->rdbuf()),
+          innerStream(std::move(inner)),
+          onDestruction(on_destruction) {
+    }
+
+    finalizing_ostream(const finalizing_ostream&) = delete;
+    finalizing_ostream& operator=(const finalizing_ostream&) = delete;
+
+    finalizing_ostream(finalizing_ostream&& other) noexcept
+        : std::ostream(std::move(other)),
+          innerStream(std::move(other.innerStream)),
+          onDestruction(std::move(other.onDestruction)) {
+        other.onDestruction = nullptr;
+    }
+
+    finalizing_ostream& operator=(finalizing_ostream&& other) noexcept {
+        if (this != &other) {
+            std::ostream::operator=(std::move(other));
+            innerStream = std::move(other.innerStream);
+            onDestruction = std::move(other.onDestruction);
+            other.onDestruction = nullptr;
+        }
+        return *this;
+    }
+
+    ~finalizing_ostream() {
+        if (onDestruction) {
+            onDestruction(std::move(innerStream));
+        }
+    }
+
+private:
+    std::unique_ptr<std::ostream> innerStream;
+    std::function<void(std::unique_ptr<std::ostream>)> onDestruction;
+};

src/io/memory_istream.hpp

@@ -32,3 +32,33 @@ public:
 private:
     memory_streambuf buf;
 };
+
+class memory_view_streambuf : public std::streambuf {
+public:
+    explicit memory_view_streambuf(const util::Buffer<char>& buffer)
+        : buffer(std::move(buffer)) {
+        char* base = const_cast<char*>(this->buffer.data());
+        char* end = base + this->buffer.size();
+        setg(base, base, end);
+    }
+    
+    memory_view_streambuf(const memory_view_streambuf&) = delete;
+    memory_view_streambuf& operator=(const memory_view_streambuf&) = delete;
+
+protected:
+    int_type underflow() override {
+        return traits_type::eof();
+    }
+
+private:
+    const util::Buffer<char>& buffer;
+};
+
+class memory_view_istream : public std::istream {
+public:
+    explicit memory_view_istream(const util::Buffer<char>& buffer)
+        : std::istream(&buf), buf(buffer) {}
+
+private:
+    memory_view_streambuf buf;
+};

src/io/memory_ostream.hpp

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <iostream>
 #include <streambuf>
 #include <cstring>

src/logic/scripting/lua/libs/libapp.cpp

@@ -1,6 +1,25 @@
 #include "api_lua.hpp"
 
+#include "io/io.hpp"
+#include "io/devices/MemoryDevice.hpp"
+
+static int l_create_memory_device(lua::State* L) {
+    std::string name = lua::require_string(L, 1);
+    if (io::get_device(name)) {
+        throw std::runtime_error(
+            "entry-point '" + name + "' is already used"
+        );
+    }
+    if (name.find(':') != std::string::npos) {
+        throw std::runtime_error("invalid entry point name");
+    }
+    
+    io::set_device(name, std::make_unique<io::MemoryDevice>());
+    return 0;
+}
+
 const luaL_Reg applib[] = {
+    {"create_memory_device", lua::wrap<l_create_memory_device>},
     // see libcore.cpp an stdlib.lua
     {nullptr, nullptr}
 };

src/logic/scripting/lua/libs/libfile.cpp

@@ -5,6 +5,7 @@
 #include "engine/Engine.hpp"
 #include "engine/EnginePaths.hpp"
 #include "io/io.hpp"
+#include "io/devices/MemoryDevice.hpp"
 #include "io/devices/ZipFileDevice.hpp"
 #include "util/stringutil.hpp"
 #include "api_lua.hpp"
@@ -49,6 +50,14 @@ static bool is_writeable(const std::string& entryPoint) {
     if (entryPoint.substr(0, 2) == "W.") {
         return true;
     }
+    // todo: do better
+    auto device = io::get_device(entryPoint);
+    if (device == nullptr) {
+        return false;
+    }
+    if (dynamic_cast<io::MemoryDevice*>(device.get())) {
+        return true;
+    }
     if (writeable_entry_points.find(entryPoint) != writeable_entry_points.end()) {
         return true;
     }
@@ -221,6 +230,16 @@ static int l_unmount(lua::State* L) {
     return 0;
 }
 
+static int l_create_memory_device(lua::State* L) {
+    if (lua::isstring(L, 1)) {
+        throw std::runtime_error(
+            "name must not be specified, use app.create_memory_device instead"
+        );
+    }
+    auto& paths = engine->getPaths();
+    return lua::pushstring(L, paths.createMemoryDevice());
+}
+
 static int l_create_zip(lua::State* L) {
     io::path folder = lua::require_string(L, 1);
     io::path outFile = lua::require_string(L, 2);
@@ -336,7 +355,6 @@ static int l_write_descriptor(lua::State* L) {
     if (!stream->good()) {
         throw std::runtime_error("failed to write to stream");
     }
-
     return 0;
 }
 
@@ -352,7 +370,6 @@ static int l_flush_descriptor(lua::State* L) {
     }
 
     scripting::descriptors_manager::flush(descriptor);
-
     return 0;
 }
 
@@ -364,13 +381,11 @@ static int l_close_descriptor(lua::State* L) {
     }
 
     scripting::descriptors_manager::close(descriptor);
-
     return 0;
 }
 
 static int l_close_all_descriptors(lua::State* L) {
     scripting::descriptors_manager::close_all_descriptors();
-
     return 0;
 }
 
@@ -396,6 +411,7 @@ const luaL_Reg filelib[] = {
     {"is_writeable", lua::wrap<l_is_writeable>},
     {"mount", lua::wrap<l_mount>},
     {"unmount", lua::wrap<l_unmount>},
+    {"create_memory_device", lua::wrap<l_create_memory_device>},
     {"create_zip", lua::wrap<l_create_zip>},
     {"__open_descriptor", lua::wrap<l_open_descriptor>},
     {"__has_descriptor", lua::wrap<l_has_descriptor>},