diff --git a/dev/tests/filesystem.lua b/dev/tests/filesystem.lua index a9ff58ce..38e7c4a0 100644 --- a/dev/tests/filesystem.lua +++ b/dev/tests/filesystem.lua @@ -4,6 +4,7 @@ assert(file.exists("config:")) debug.log("write text file") assert(file.write("config:text.txt", "example, пример")) assert(file.exists("config:text.txt")) +assert(file.isfile("config:text.txt")) debug.log("read text file") assert(file.read("config:text.txt") == "example, пример") @@ -18,11 +19,23 @@ assert(file.isdir("config:dir")) debug.log("remove directory") file.remove("config:dir") +assert(not file.isdir("config:dir")) debug.log("create directories") file.mkdirs("config:dir/subdir/other") assert(file.isdir("config:dir/subdir/other")) +debug.log("list directory") +file.write("config:dir/subdir/a.txt", "helloworld") +file.write("config:dir/subdir/b.txt", "gfgsfhs") + +local entries = file.list("config:dir/subdir") +assert(#entries == 3) +table.sort(entries) +assert(entries[1] == "config:dir/subdir/a.txt") +assert(entries[2] == "config:dir/subdir/b.txt") +assert(entries[3] == "config:dir/subdir/other") + debug.log("remove tree") file.remove_tree("config:dir") assert(not file.isdir("config:dir")) diff --git a/dev/tests/memory_filesystem.lua b/dev/tests/memory_filesystem.lua new file mode 100644 index 00000000..e246279a --- /dev/null +++ b/dev/tests/memory_filesystem.lua @@ -0,0 +1,67 @@ +app.create_memory_device("memtest") +local tmp = file.create_memory_device() + +debug.log("check initial state") +assert(file.exists("memtest:")) +assert(file.is_writeable("memtest:")) +assert(file.is_writeable(tmp..":")) + +debug.log("write text file") +assert(file.write("memtest:text.txt", "example, пример")) +assert(file.exists("memtest:text.txt")) +assert(file.isfile("memtest:text.txt")) + +debug.log("read text file") +assert(file.read("memtest:text.txt") == "example, пример") + +debug.log("delete file") +file.remove("memtest:text.txt") +assert(not file.exists("memtest:text.txt")) + +debug.log("create directory") +file.mkdir("memtest:dir") +assert(file.isdir("memtest:dir")) + +debug.log("remove directory") +file.remove("memtest:dir") +assert(not file.isdir("memtest:dir")) + +debug.log("create directories") +file.mkdirs("memtest:dir/subdir/other") +assert(file.isdir("memtest:dir/subdir/other")) + +debug.log("list directory") +file.write("memtest:dir/subdir/a.txt", "helloworld") +file.write("memtest:dir/subdir/b.txt", "gfgsfhs") + +local entries = file.list("memtest:dir/subdir") +assert(#entries == 3) +table.sort(entries) +assert(entries[1] == "memtest:dir/subdir/a.txt") +assert(entries[2] == "memtest:dir/subdir/b.txt") +assert(entries[3] == "memtest:dir/subdir/other") + +debug.log("remove tree") +file.remove_tree("memtest:dir") +assert(not file.isdir("memtest:dir")) + +debug.log("write binary file") +local bytes = {0xDE, 0xAD, 0xC0, 0xDE} +file.write_bytes("memtest:binary", bytes) +assert(file.exists("memtest:binary")) + +debug.log("write binary file") +local bytes = {0xDE, 0xAD, 0xC0, 0xDE} +file.write_bytes("memtest:binary", bytes) +assert(file.exists("memtest:binary")) + +debug.log("read binary file") +local rbytes = file.read_bytes("memtest:binary") +assert(#rbytes == #bytes) +for i, b in ipairs(bytes) do + assert(rbytes[i] == b) +end + +debug.log("delete file") +file.remove("memtest:binary") +assert(not file.exists("memtest:binary")) diff --git a/doc/en/scripting/builtins/libapp.md b/doc/en/scripting/builtins/libapp.md index 45414ba1..2dd78361 100644 --- a/doc/en/scripting/builtins/libapp.md +++ b/doc/en/scripting/builtins/libapp.md @@ -158,3 +158,13 @@ app.get_setting_info(name: str) -> { ``` Returns a table with information about a setting. Throws an exception if the setting does not exist. + + +```lua +app.create_memory_device( + -- entry-point name + name: str +) +``` + +Creates an in-memory filesystem. diff --git a/doc/en/scripting/builtins/libfile.md b/doc/en/scripting/builtins/libfile.md index 73d286af..0d4374ca 100644 --- a/doc/en/scripting/builtins/libfile.md +++ b/doc/en/scripting/builtins/libfile.md @@ -139,6 +139,12 @@ file.create_zip(directory: str, output_file: str) --> str Creates a ZIP archive from the contents of the specified directory. +```lua +file.create_memory_device() --> str +``` + +Creates a memory file system and returns entry point name. Lives until content unload. + ```lua file.name(path: str) --> str ``` diff --git a/doc/ru/scripting/builtins/libapp.md b/doc/ru/scripting/builtins/libapp.md index d2ddec72..ddd91f14 100644 --- a/doc/ru/scripting/builtins/libapp.md +++ b/doc/ru/scripting/builtins/libapp.md @@ -159,3 +159,12 @@ app.get_setting_info(name: str) -> { ``` Возвращает таблицу с информацией о настройке. Бросает исключение, если настройки не существует. + +```lua +app.create_memory_device( + -- имя точки входа + name: str +) +``` + +Создаёт файловую систему в памяти. diff --git a/doc/ru/scripting/builtins/libfile.md b/doc/ru/scripting/builtins/libfile.md index 4afbe23c..67af5178 100644 --- a/doc/ru/scripting/builtins/libfile.md +++ b/doc/ru/scripting/builtins/libfile.md @@ -139,6 +139,12 @@ file.create_zip(директория: str, выходной_файл: str) --> s Создаёт ZIP-архив из содержимого указанной директории. +```lua +file.create_memory_device() --> str +``` + +Создаёт файловую систему в памяти, возвращает имя точки входа. Удаляется при выгрузке контента. + ```lua file.name(путь: str) --> str ``` diff --git a/src/engine/EnginePaths.cpp b/src/engine/EnginePaths.cpp index 8a786140..82aea253 100644 --- a/src/engine/EnginePaths.cpp +++ b/src/engine/EnginePaths.cpp @@ -2,6 +2,7 @@ #include "debug/Logger.hpp" #include "io/devices/StdfsDevice.hpp" +#include "io/devices/MemoryDevice.hpp" #include "io/devices/ZipFileDevice.hpp" #include "maths/util.hpp" #include "typedefs.hpp" @@ -168,6 +169,18 @@ void EnginePaths::unmount(const std::string& name) { mounted.erase(found); } +std::string EnginePaths::createMemoryDevice() { + auto device = std::make_unique(); + std::string name; + do { + name = std::string("W.") + generate_random_base64<6>(); + } while (std::find(mounted.begin(), mounted.end(), name) != mounted.end()); + + io::set_device(name, std::move(device)); + mounted.push_back(name); + return name; +} + std::string EnginePaths::createWriteableDevice(const std::string& name) { const auto& found = writeables.find(name); if (found != writeables.end()) { diff --git a/src/engine/EnginePaths.hpp b/src/engine/EnginePaths.hpp index 41d02236..8e5f71a2 100644 --- a/src/engine/EnginePaths.hpp +++ b/src/engine/EnginePaths.hpp @@ -58,6 +58,7 @@ public: void unmount(const std::string& name); std::string createWriteableDevice(const std::string& name); + std::string createMemoryDevice(); void setEntryPoints(std::vector entryPoints); diff --git a/src/io/deflate_ostream.hpp b/src/io/deflate_ostream.hpp index 39678843..a029e021 100644 --- a/src/io/deflate_ostream.hpp +++ b/src/io/deflate_ostream.hpp @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/io/devices/MemoryDevice.cpp b/src/io/devices/MemoryDevice.cpp new file mode 100644 index 00000000..f9f4760e --- /dev/null +++ b/src/io/devices/MemoryDevice.cpp @@ -0,0 +1,220 @@ +#include "MemoryDevice.hpp" + +#include "../memory_istream.hpp" +#include "../memory_ostream.hpp" +#include "../finalizing_ostream.hpp" + +#include + +io::MemoryDevice::MemoryDevice() {} + +std::filesystem::path io::MemoryDevice::resolve(std::string_view path) { + throw std::runtime_error("unable to resolve filesystem path"); +} + +std::unique_ptr io::MemoryDevice::write(std::string_view path) { + std::string filePath = std::string(path); + return std::make_unique( + std::make_unique(), + [this, filePath](auto ostream) { + auto& memoryStream = dynamic_cast(*ostream); + createFile(std::move(filePath), memoryStream.release()); + } + ); +} + +std::unique_ptr io::MemoryDevice::read(std::string_view path) { + const auto& found = nodes.find(std::string(path)); + if (found == nodes.end()) { + return nullptr; + } + auto& node = found->second; + if (auto file = node.get_if()) { + if (file->content != nullptr) { + return std::make_unique(file->content); + } + } + return nullptr; +} + +size_t io::MemoryDevice::size(std::string_view path) { + const auto& found = nodes.find(std::string(path)); + if (found == nodes.end()) { + return 0; + } + return std::visit([](auto&& arg) -> size_t { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return arg.content.size(); + } else if constexpr (std::is_same_v) { + return arg.content.size(); + } else { + return 0; + } + }, found->second.data); +} + +io::file_time_type io::MemoryDevice::lastWriteTime(std::string_view path) { + return file_time_type::min(); +} + +bool io::MemoryDevice::exists(std::string_view path) { + if (path.empty()) { + return true; + } + return nodes.find(std::string(path)) != nodes.end(); +} + +bool io::MemoryDevice::isdir(std::string_view path) { + if (path.empty()) { + return true; + } + const auto& found = nodes.find(std::string(path)); + if (found == nodes.end()) { + return false; + } + return found->second.holds_alternative(); +} + +bool io::MemoryDevice::isfile(std::string_view path) { + const auto& found = nodes.find(std::string(path)); + if (found == nodes.end()) { + return false; + } + return found->second.holds_alternative(); +} + +bool io::MemoryDevice::mkdir(std::string_view path) { + return createDir(std::string(path)) != nullptr; +} + +bool io::MemoryDevice::mkdirs(std::string_view path) { + io::path dirPath = std::string(path); + std::vector parts; + while (!dirPath.pathPart().empty()) { + parts.push_back(dirPath.name()); + dirPath = dirPath.parent(); + } + for (int i = parts.size() - 1; i >= 0; i--) { + dirPath = dirPath / parts[i]; + createDir(dirPath.string()); + } + return true; +} + +bool io::MemoryDevice::remove(std::string_view path) { + std::string pathString = std::string(path); + const auto& found = nodes.find(pathString); + if (found == nodes.end()) { + return false; + } + if (found->second.holds_alternative()) { + const auto& dir = found->second.get_if(); + if (!dir->content.empty()) { + return false; + } + } + io::path filePath = pathString; + io::path parentPath = filePath.parent(); + auto parentDir = getDir(parentPath.string()); + if (parentDir) { + auto& content = parentDir->content; + content.erase( + std::remove(content.begin(), content.end(), filePath.name()), + content.end() + ); + } + nodes.erase(found); + return true; +} + +uint64_t io::MemoryDevice::removeAll(std::string_view path) { + std::string pathString = std::string(path); + const auto& found = nodes.find(pathString); + if (found == nodes.end()) { + return 0; + } + io::path filePath = pathString; + + uint64_t count = 0; + if (found->second.holds_alternative()) { + auto dir = found->second.get_if(); + auto files = dir->content; + for (const auto& name : files) { + io::path subPath = filePath / name; + count += removeAll(subPath.string()); + } + } + if (remove(pathString)) { + count++; + } + return count; +} + +namespace { + struct MemoryPathsGenerator : public io::PathsGenerator { + std::vector entries; + size_t index = 0; + + MemoryPathsGenerator(std::vector&& entries) + : entries(std::move(entries)) {} + + bool next(io::path& outPath) override { + if (index >= entries.size()) { + return false; + } + outPath = entries[index++]; + return true; + } + }; +} + +std::unique_ptr io::MemoryDevice::list(std::string_view path) { + auto dir = getDir(path); + if (!dir) { + return nullptr; + } + return std::make_unique( + std::vector(dir->content) + ); +} + +io::MemoryDevice::Dir* io::MemoryDevice::createDir(std::string path) { + io::path filePath = path; + io::path parent = filePath.parent(); + auto parentDir = getDir(parent.string()); + if (!parentDir) { + return nullptr; + } + parentDir->content.push_back(filePath.name()); + auto& node = nodes[std::move(path)]; + node.data = Dir {}; + return node.get_if(); +} + +io::MemoryDevice::Node* io::MemoryDevice::createFile( + std::string path, util::Buffer&& content +) { + io::path filePath = path; + io::path parent = filePath.parent(); + auto dir = getDir(parent.string()); + if (!dir) { + return nullptr; + } + dir->content.push_back(filePath.name()); + auto& node = nodes[std::move(path)]; + node.data = File {std::move(content)}; + return &node; +} + +io::MemoryDevice::Dir* io::MemoryDevice::getDir(std::string_view path) { + if (path.empty()) { + return &rootDir; + } + const auto& found = nodes.find(std::string(path)); + if (found == nodes.end()) { + return nullptr; + } + auto& node = found->second; + return node.get_if(); +} diff --git a/src/io/devices/MemoryDevice.hpp b/src/io/devices/MemoryDevice.hpp new file mode 100644 index 00000000..d6f97c65 --- /dev/null +++ b/src/io/devices/MemoryDevice.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include "Device.hpp" +#include "util/Buffer.hpp" + +#include +#include +#include +#include + +namespace io { + /// @brief In-memory filesystem device + class MemoryDevice : public Device { + enum class NodeType { + DIR, FILE + }; + + struct File { + util::Buffer content; + }; + + struct Dir { + std::vector content; + }; + + struct Node { + std::variant data; + + NodeType type() const { + return std::visit([](auto&& arg) -> NodeType { + using T = std::decay_t; + if constexpr (std::is_same_v) return NodeType::DIR; + else if constexpr (std::is_same_v) return NodeType::FILE; + }, data); + } + + template + bool holds_alternative() const { + return std::holds_alternative(data); + } + + template + T* get_if() { + return std::get_if(&data); + } + }; + public: + MemoryDevice(); + + std::filesystem::path resolve(std::string_view path) override; + std::unique_ptr write(std::string_view path) override; + std::unique_ptr read(std::string_view path) override; + size_t size(std::string_view path) override; + file_time_type lastWriteTime(std::string_view path) override; + bool exists(std::string_view path) override; + bool isdir(std::string_view path) override; + bool isfile(std::string_view path) override; + bool mkdir(std::string_view path) override; + bool mkdirs(std::string_view path) override; + bool remove(std::string_view path) override; + uint64_t removeAll(std::string_view path) override; + std::unique_ptr list(std::string_view path) override; + private: + std::unordered_map nodes; + Dir rootDir {}; + + Node* createFile(std::string path, util::Buffer&& content); + Dir* createDir(std::string path); + Dir* getDir(std::string_view path); + }; +} diff --git a/src/io/devices/StdfsDevice.hpp b/src/io/devices/StdfsDevice.hpp index 69b12234..677cad25 100644 --- a/src/io/devices/StdfsDevice.hpp +++ b/src/io/devices/StdfsDevice.hpp @@ -1,3 +1,5 @@ +#pragma once + #include "Device.hpp" namespace io { diff --git a/src/io/devices/ZipFileDevice.cpp b/src/io/devices/ZipFileDevice.cpp index 80021ba2..dce05388 100644 --- a/src/io/devices/ZipFileDevice.cpp +++ b/src/io/devices/ZipFileDevice.cpp @@ -239,7 +239,7 @@ std::unique_ptr ZipFileDevice::read(std::string_view path) { size_t ZipFileDevice::size(std::string_view path) { const auto& found = entries.find(std::string(path)); if (found == entries.end()) { - return false; + return 0; } return found->second.uncompressedSize; } diff --git a/src/io/finalizing_ostream.hpp b/src/io/finalizing_ostream.hpp new file mode 100644 index 00000000..c8462247 --- /dev/null +++ b/src/io/finalizing_ostream.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +class finalizing_ostream final : public std::ostream { +public: + finalizing_ostream( + std::unique_ptr inner, + std::function)> on_destruction + ) + : std::ostream(inner->rdbuf()), + innerStream(std::move(inner)), + onDestruction(on_destruction) { + } + + finalizing_ostream(const finalizing_ostream&) = delete; + finalizing_ostream& operator=(const finalizing_ostream&) = delete; + + finalizing_ostream(finalizing_ostream&& other) noexcept + : std::ostream(std::move(other)), + innerStream(std::move(other.innerStream)), + onDestruction(std::move(other.onDestruction)) { + other.onDestruction = nullptr; + } + + finalizing_ostream& operator=(finalizing_ostream&& other) noexcept { + if (this != &other) { + std::ostream::operator=(std::move(other)); + innerStream = std::move(other.innerStream); + onDestruction = std::move(other.onDestruction); + other.onDestruction = nullptr; + } + return *this; + } + + ~finalizing_ostream() { + if (onDestruction) { + onDestruction(std::move(innerStream)); + } + } + +private: + std::unique_ptr innerStream; + std::function)> onDestruction; +}; diff --git a/src/io/memory_istream.hpp b/src/io/memory_istream.hpp index 0b2131d6..7c0aded6 100644 --- a/src/io/memory_istream.hpp +++ b/src/io/memory_istream.hpp @@ -32,3 +32,33 @@ public: private: memory_streambuf buf; }; + +class memory_view_streambuf : public std::streambuf { +public: + explicit memory_view_streambuf(const util::Buffer& buffer) + : buffer(std::move(buffer)) { + char* base = const_cast(this->buffer.data()); + char* end = base + this->buffer.size(); + setg(base, base, end); + } + + memory_view_streambuf(const memory_view_streambuf&) = delete; + memory_view_streambuf& operator=(const memory_view_streambuf&) = delete; + +protected: + int_type underflow() override { + return traits_type::eof(); + } + +private: + const util::Buffer& buffer; +}; + +class memory_view_istream : public std::istream { +public: + explicit memory_view_istream(const util::Buffer& buffer) + : std::istream(&buf), buf(buffer) {} + +private: + memory_view_streambuf buf; +}; diff --git a/src/io/memory_ostream.hpp b/src/io/memory_ostream.hpp index d8849704..3ff14a99 100644 --- a/src/io/memory_ostream.hpp +++ b/src/io/memory_ostream.hpp @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/logic/scripting/lua/libs/libapp.cpp b/src/logic/scripting/lua/libs/libapp.cpp index 4bd38f4b..fae18bb6 100644 --- a/src/logic/scripting/lua/libs/libapp.cpp +++ b/src/logic/scripting/lua/libs/libapp.cpp @@ -1,6 +1,25 @@ #include "api_lua.hpp" +#include "io/io.hpp" +#include "io/devices/MemoryDevice.hpp" + +static int l_create_memory_device(lua::State* L) { + std::string name = lua::require_string(L, 1); + if (io::get_device(name)) { + throw std::runtime_error( + "entry-point '" + name + "' is already used" + ); + } + if (name.find(':') != std::string::npos) { + throw std::runtime_error("invalid entry point name"); + } + + io::set_device(name, std::make_unique()); + return 0; +} + const luaL_Reg applib[] = { + {"create_memory_device", lua::wrap}, // see libcore.cpp an stdlib.lua {nullptr, nullptr} }; diff --git a/src/logic/scripting/lua/libs/libfile.cpp b/src/logic/scripting/lua/libs/libfile.cpp index c3ee4bdb..197cd7db 100644 --- a/src/logic/scripting/lua/libs/libfile.cpp +++ b/src/logic/scripting/lua/libs/libfile.cpp @@ -5,6 +5,7 @@ #include "engine/Engine.hpp" #include "engine/EnginePaths.hpp" #include "io/io.hpp" +#include "io/devices/MemoryDevice.hpp" #include "io/devices/ZipFileDevice.hpp" #include "util/stringutil.hpp" #include "api_lua.hpp" @@ -49,6 +50,14 @@ static bool is_writeable(const std::string& entryPoint) { if (entryPoint.substr(0, 2) == "W.") { return true; } + // todo: do better + auto device = io::get_device(entryPoint); + if (device == nullptr) { + return false; + } + if (dynamic_cast(device.get())) { + return true; + } if (writeable_entry_points.find(entryPoint) != writeable_entry_points.end()) { return true; } @@ -221,6 +230,16 @@ static int l_unmount(lua::State* L) { return 0; } +static int l_create_memory_device(lua::State* L) { + if (lua::isstring(L, 1)) { + throw std::runtime_error( + "name must not be specified, use app.create_memory_device instead" + ); + } + auto& paths = engine->getPaths(); + return lua::pushstring(L, paths.createMemoryDevice()); +} + static int l_create_zip(lua::State* L) { io::path folder = lua::require_string(L, 1); io::path outFile = lua::require_string(L, 2); @@ -336,7 +355,6 @@ static int l_write_descriptor(lua::State* L) { if (!stream->good()) { throw std::runtime_error("failed to write to stream"); } - return 0; } @@ -352,7 +370,6 @@ static int l_flush_descriptor(lua::State* L) { } scripting::descriptors_manager::flush(descriptor); - return 0; } @@ -364,13 +381,11 @@ static int l_close_descriptor(lua::State* L) { } scripting::descriptors_manager::close(descriptor); - return 0; } static int l_close_all_descriptors(lua::State* L) { scripting::descriptors_manager::close_all_descriptors(); - return 0; } @@ -396,6 +411,7 @@ const luaL_Reg filelib[] = { {"is_writeable", lua::wrap}, {"mount", lua::wrap}, {"unmount", lua::wrap}, + {"create_memory_device", lua::wrap}, {"create_zip", lua::wrap}, {"__open_descriptor", lua::wrap}, {"__has_descriptor", lua::wrap},