Merge pull request #326 from MihailRis/vec3-models-loader

VEC3 models loader

doc/specs/vec3_model_spec.md

@@ -0,0 +1,109 @@
+# VEC3 format specification
+
+3D models binary format.
+
+Byteorder: little-endian
+
+## Syntax
+
+```cpp
+enum AttributeType:uint8 {
+    POSITION = 0,
+    UV,
+    NORMAL,
+    COLOR,
+};
+sizeof(AttributeType) == 1;
+
+struct VertexAttribute {
+    AttributeType type; // data type is infered from attribute type
+    uint8 flags;
+    uint32 size;
+    float data[]; // if compressed, first 4 bytes of compressed data is decompressed size
+};
+sizeof(VertexAttribute) == 6; // + dynamic data array
+
+struct Mesh {
+    uint32 triangle_count; // number of mesh triangles
+    uint16 material_id;
+    uint16 flags;
+    uint16 attribute_count;
+    VertexAttribute attributes[];
+    uint8 indices[]; // if compressed, first 4 bytes of compressed data is compressed buffer size
+};
+sizeof(Mesh) == 10; // + dynamic attributes array + dynamic indices array
+
+struct Model {
+	uint16 name_len;
+    vec3 origin;
+    uint32 mesh_count;
+    Mesh meshes[];
+    char name[];
+};
+sizeof(Model) == 18; // + dynamic Mesh array + name length
+
+struct Material {
+    uint16 flags;
+    uint16 name_len;
+    char name[];
+};
+sizeof(Material) == 4; // + dynamic sized string
+
+struct Header {
+    char[8] ident;   // "\0\0VEC3\0\0"
+    uint16 version;  // current is 1
+    uint16 reserved; // 0x0000
+};
+sizeof(Header) == 12;
+
+struct Body {
+    uint16 material_count
+    uint16 model_count
+    Material materials[];
+    Model models[];
+};
+sizeof(Body) == 4; // + dynamic models array + dynamic materials array
+
+```
+
+\* vertex data: positions are global. Model origins used to make it local.
+
+vertex - is a set of vertex data section entries indices divided by stride, starting from 0 (section first entry).
+
+Example: in file having sections (coordinates, texture_coordinates, normal) vertex is a set of 3 indices ordered the
+same way as sections stored in the file.
+
+## Vertex Data section tags
+
+All sections are optional.
+
+| Value | Name                | Stride (bytes) | Description                 |
+| ----- | ------------------- | -------------- | --------------------------- |
+| %x01  | Coordinates         | 12             | vertex position             |
+| %x02  | Texture coordinates | 8              | vertex texture coordinates  |
+| %x03  | Normals             | 12             | vertex normal vector        |
+| %x04  | Color               | 16             | vertex RGBA color (0.0-1.0) |
+
+VertexAttribute flags:
+
+| Value | Name             |
+| ----- | ---------------- |
+| %x01  | ZLib compression |
+
+## Mesh
+
+Mesh flags:
+
+| Value | Name                                |
+| ----- | ----------------------------------- |
+| %x01  | Indices ZLib compression            |
+| %x02  | Use 16 bit indices instead of 8 bit |
+
+## Material
+
+Material flags:
+
+| Bit offset | Description |
+|------------|-------------|
+| 0          | Shadeless   |
+| 1-7        | Reserved    |

src/assets/AssetsLoader.cpp

@@ -228,7 +228,11 @@ void AssetsLoader::addDefaults(AssetsLoader& loader, const Content* content) {
         for (auto& entry : content->getSkeletons()) {
             auto& skeleton = *entry.second;
             for (auto& bone : skeleton.getBones()) {
-                auto& model = bone->model.name;
+                std::string model = bone->model.name;
+                size_t pos = model.rfind('.');
+                if (pos != std::string::npos) {
+                    model = model.substr(0, pos);
+                }
                 if (!model.empty()) {
                     loader.add(
                         AssetType::MODEL, MODELS_FOLDER + "/" + model, model

src/assets/assetload_funcs.cpp

@@ -10,6 +10,7 @@
 #include "coders/imageio.hpp"
 #include "coders/json.hpp"
 #include "coders/obj.hpp"
+#include "coders/vec3.hpp"
 #include "constants.hpp"
 #include "debug/Logger.hpp"
 #include "files/engine_paths.hpp"
@@ -23,6 +24,7 @@
 #include "graphics/core/Texture.hpp"
 #include "graphics/core/TextureAnimation.hpp"
 #include "objects/rigging.hpp"
+#include "util/stringutil.hpp"
 #include "Assets.hpp"
 #include "AssetsLoader.hpp"
 
@@ -39,18 +41,32 @@ static bool animation(
     Atlas* dstAtlas
 );
 
-assetload::postfunc assetload::
-    texture(AssetsLoader*, const ResPaths* paths, const std::string& filename, const std::string& name, const std::shared_ptr<AssetCfg>&) {
-    std::shared_ptr<ImageData> image(
-        imageio::read(paths->find(filename + ".png").u8string()).release()
-    );
-    return [name, image](auto assets) {
-        assets->store(Texture::from(image.get()), name);
-    };
+assetload::postfunc assetload::texture(
+    AssetsLoader*,
+    const ResPaths* paths,
+    const std::string& filename,
+    const std::string& name,
+    const std::shared_ptr<AssetCfg>&
+) {
+    auto actualFile = paths->find(filename + ".png").u8string();
+    try {
+        std::shared_ptr<ImageData> image(imageio::read(actualFile).release());
+        return [name, image, actualFile](auto assets) {
+            assets->store(Texture::from(image.get()), name);
+        };
+    } catch (const std::runtime_error& err) {
+        logger.error() << actualFile << ": " << err.what();
+        return [](auto) {};
+    }
 }
 
-assetload::postfunc assetload::
-    shader(AssetsLoader*, const ResPaths* paths, const std::string& filename, const std::string& name, const std::shared_ptr<AssetCfg>&) {
+assetload::postfunc assetload::shader(
+    AssetsLoader*,
+    const ResPaths* paths,
+    const std::string& filename,
+    const std::string& name,
+    const std::shared_ptr<AssetCfg>&
+) {
     fs::path vertexFile = paths->find(filename + ".glslv");
     fs::path fragmentFile = paths->find(filename + ".glslf");
 
@@ -181,8 +197,7 @@ assetload::postfunc assetload::sound(
         if (!fs::exists(variantFile)) {
             break;
         }
-        baseSound->variants.emplace_back(audio::load_sound(variantFile, keepPCM)
-        );
+        baseSound->variants.emplace_back(audio::load_sound(variantFile, keepPCM));
     }
 
     auto sound = baseSound.release();
@@ -191,21 +206,50 @@ assetload::postfunc assetload::sound(
     };
 }
 
-assetload::postfunc assetload::
-    model(AssetsLoader* loader, const ResPaths* paths, const std::string& file, const std::string& name, const std::shared_ptr<AssetCfg>&) {
-    auto path = paths->find(file + ".obj");
+static void request_textures(AssetsLoader* loader, const model::Model& model) {
+    for (auto& mesh : model.meshes) {
+        if (mesh.texture.find('$') == std::string::npos) {
+            auto filename = TEXTURES_FOLDER + "/" + mesh.texture;
+            loader->add(
+                AssetType::TEXTURE, filename, mesh.texture, nullptr
+            );
+        }
+    }
+}
+
+assetload::postfunc assetload::model(
+    AssetsLoader* loader,
+    const ResPaths* paths,
+    const std::string& file,
+    const std::string& name,
+    const std::shared_ptr<AssetCfg>&
+) {
+    auto path = paths->find(file + ".vec3");
+    if (fs::exists(path)) {
+        auto bytes = files::read_bytes_buffer(path);
+        auto modelVEC3 = std::make_shared<vec3::File>(vec3::load(path.u8string(), bytes));
+        return [loader, name, modelVEC3=std::move(modelVEC3)](Assets* assets) {
+            for (auto& [modelName, model] : modelVEC3->models) {
+                request_textures(loader, model.model);
+                std::string fullName = name;
+                if (name != modelName) {
+                    fullName += "." + modelName;
+                }
+                assets->store(
+                    std::make_unique<model::Model>(model.model),
+                    fullName
+                );
+                logger.info() << "store model " << util::quote(modelName)
+                              << " as " << util::quote(fullName);
+            }
+        };
+    }
+    path = paths->find(file + ".obj");
     auto text = files::read_string(path);
     try {
         auto model = obj::parse(path.u8string(), text).release();
         return [=](Assets* assets) {
-            for (auto& mesh : model->meshes) {
-                if (mesh.texture.find('$') == std::string::npos) {
-                    auto filename = TEXTURES_FOLDER + "/" + mesh.texture;
-                    loader->add(
-                        AssetType::TEXTURE, filename, mesh.texture, nullptr
-                    );
-                }
-            }
+            request_textures(loader, *model);
             assets->store(std::unique_ptr<model::Model>(model), name);
         };
     } catch (const parsing_error& err) {

src/coders/byte_utils.cpp

@@ -107,6 +107,14 @@ void ByteReader::checkMagic(const char* data, size_t size) {
     pos += size;
 }
 
+void ByteReader::get(char* dst, size_t size) {
+    if (pos + size > this->size) {
+        throw std::runtime_error("buffer underflow");
+    }
+    std::memcpy(dst, data+pos, size);
+    pos += size;
+}
+
 ubyte ByteReader::get() {
     if (pos == size) {
         throw std::runtime_error("buffer underflow");

src/coders/byte_utils.hpp

@@ -52,6 +52,8 @@ public:
     ByteReader(const ubyte* data);
 
     void checkMagic(const char* data, size_t size);
+    /// @brief Get N bytes
+    void get(char* dst, size_t size);
     /// @brief Read one byte (unsigned 8 bit integer)
     ubyte get();
     /// @brief Read one byte (unsigned 8 bit integer) without pointer move

src/coders/vec3.cpp

@@ -0,0 +1,224 @@
+#include "vec3.hpp"
+
+#include <stdexcept>
+
+#include "byte_utils.hpp"
+#include "util/data_io.hpp"
+#include "util/stringutil.hpp"
+#include "graphics/core/Model.hpp"
+
+inline constexpr int VERSION = 1;
+
+inline constexpr int FLAG_ZLIB = 0x1;
+inline constexpr int FLAG_16BIT_INDICES = 0x2;
+
+using namespace vec3;
+
+vec3::Model::~Model() = default;
+
+enum AttributeType {
+    POSITION = 0,
+    UV,
+    NORMAL,
+    COLOR
+};
+
+struct VertexAttribute {
+    AttributeType type;
+    int flags;
+    util::Buffer<float> data;
+
+    VertexAttribute() = default;
+
+    VertexAttribute(VertexAttribute&&) = default;
+
+    VertexAttribute& operator=(VertexAttribute&& o) {
+        type = o.type;
+        flags = o.flags;
+        data = std::move(o.data);
+        return *this;
+    }
+};
+
+static VertexAttribute load_attribute(ByteReader& reader) {
+    auto type = static_cast<AttributeType>(reader.get());
+    int flags = reader.get();
+    assert(type >= POSITION && flags <= COLOR);
+    if (flags != 0) {
+        throw std::runtime_error("attribute compression is not supported yet");
+    }
+    int size = reader.getInt32();
+
+    util::Buffer<float> data(size / sizeof(float));
+    reader.get(reinterpret_cast<char*>(data.data()), size);
+    if (dataio::is_big_endian()) {
+        for (int i = 0; i < data.size(); i++) {
+            data[i] = dataio::swap(data[i]);
+        }
+    }
+    return VertexAttribute {type, flags, std::move(data)};
+}
+
+static model::Mesh build_mesh(
+    const std::vector<VertexAttribute>& attrs, 
+    const util::Buffer<uint16_t>& indices,
+    const std::string& texture
+) {
+    const glm::vec3* coords = nullptr;
+    const glm::vec2* uvs = nullptr;
+    const glm::vec3* normals = nullptr;
+
+    int coordsIndex, uvsIndex, normalsIndex;
+
+    for (int i = 0; i < attrs.size(); i++) {
+        const auto& attr = attrs[i];
+        switch (attr.type) {
+            case POSITION:
+                coords = reinterpret_cast<const glm::vec3*>(attr.data.data());
+                coordsIndex = i;
+                break;
+            case UV:
+                uvs = reinterpret_cast<const glm::vec2*>(attr.data.data());
+                uvsIndex = i;
+                break;
+            case NORMAL:
+                normals = reinterpret_cast<const glm::vec3*>(attr.data.data());
+                normalsIndex = i;
+                break;
+            case COLOR: // unused
+                break;
+        }
+    }
+    std::vector<model::Vertex> vertices;
+    int attrsCount = attrs.size();
+    int verticesCount = indices.size() / attrsCount;
+    for (int i = 0; i < verticesCount; i++) {
+        model::Vertex vertex {};
+        if (coords) {
+            vertex.coord = coords[indices[i * attrsCount + coordsIndex]];
+        }
+        if (uvs) {
+            vertex.uv = uvs[indices[i * attrsCount + uvsIndex]];
+        }
+        if (normals) {
+            vertex.normal = normals[indices[i * attrsCount + normalsIndex]];
+        } else if (coords) {
+            // Flat normal calculation
+            int idx = (i / 3) * 3;
+            auto a = coords[indices[idx * attrsCount + coordsIndex]];
+            auto b = coords[indices[(idx + 1) * attrsCount + coordsIndex]];
+            auto c = coords[indices[(idx + 2) * attrsCount + coordsIndex]];
+            vertex.normal = glm::normalize(glm::cross(b - a, c - a));
+        }
+        vertices.push_back(std::move(vertex));
+    }
+    return model::Mesh {texture, std::move(vertices)};
+}
+
+static model::Mesh load_mesh(
+    ByteReader& reader, const std::vector<Material>& materials
+) {
+    int triangleCount = reader.getInt32();
+    int materialId = reader.getInt16();
+    int flags = reader.getInt16();
+    int attributeCount = reader.getInt16();
+    if (flags == FLAG_ZLIB) {
+        throw std::runtime_error("compression is not supported yet");
+    }
+    assert(flags == 0);
+    std::vector<VertexAttribute> attributes;
+    for (int i = 0; i < attributeCount; i++) {
+        attributes.push_back(load_attribute(reader));
+    }
+
+    util::Buffer<uint16_t> indices(triangleCount * 3 * attributeCount);
+    if ((flags & FLAG_16BIT_INDICES) == 0){
+        util::Buffer<uint8_t> smallIndices(indices.size());
+        reader.get(
+            reinterpret_cast<char*>(smallIndices.data()),
+            indices.size() * sizeof(uint8_t)
+        );
+        for (int i = 0; i < indices.size(); i++) {
+            indices[i] = smallIndices[i];
+        }
+    } else {
+        reader.get(
+            reinterpret_cast<char*>(indices.data()),
+            indices.size() * sizeof(uint16_t)
+        );
+    }
+    if (dataio::is_big_endian()) {
+        for (int i = 0; i < indices.size(); i++) {
+            indices[i] = dataio::swap(indices[i]);
+        }
+    }
+    return build_mesh(
+        attributes,
+        indices,
+        materials.at(materialId).name
+    );
+}
+
+static Model load_model(
+    ByteReader& reader, const std::vector<Material>& materials
+) {
+    int nameLength = reader.getInt16();
+    assert(nameLength >= 0);
+    float x = reader.getFloat32();
+    float y = reader.getFloat32();
+    float z = reader.getFloat32();
+    int meshCount = reader.getInt32();
+    assert(meshCount >= 0);
+    
+    std::vector<model::Mesh> meshes;
+    for (int i = 0; i < meshCount; i++) {
+        meshes.push_back(load_mesh(reader, materials));
+    }
+    util::Buffer<char> chars(nameLength);
+    reader.get(chars.data(), nameLength);
+    std::string name(chars.data(), nameLength);
+    return Model {std::move(name), model::Model {std::move(meshes)}, {x, y, z}};
+}
+
+static Material load_material(ByteReader& reader) {
+    int flags = reader.getInt16();
+    int nameLength = reader.getInt16();
+    assert(nameLength >= 0);
+    util::Buffer<char> chars(nameLength);
+    reader.get(chars.data(), nameLength);
+    std::string name(chars.data(), nameLength);
+    return Material {flags, std::move(name)};
+}
+
+File vec3::load(
+    const std::string_view file, const util::Buffer<ubyte>& src
+) {
+    ByteReader reader(src.data(), src.size());
+    
+    // Header
+    reader.checkMagic("\0\0VEC3\0\0", 8);
+    int version = reader.getInt16();
+    int reserved = reader.getInt16();
+    if (version > VERSION) {
+        throw std::runtime_error("unsupported VEC3 version");
+    }
+    assert(reserved == 0);
+    
+    // Body
+    int materialCount = reader.getInt16();
+    int modelCount = reader.getInt16();
+    assert(materialCount >= 0);
+    assert(modelCount >= 0);
+
+    std::vector<Material> materials;
+    for (int i = 0; i < materialCount; i++) {
+        materials.push_back(load_material(reader));
+    }
+
+    std::unordered_map<std::string, Model> models;
+    for (int i = 0; i < modelCount; i++) {
+        Model model = load_model(reader, materials);
+        models[model.name] = std::move(model);
+    }
+    return File {std::move(models), std::move(materials)};
+}

src/coders/vec3.hpp

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+#include <glm/glm.hpp>
+#include <unordered_map>
+
+#include "typedefs.hpp"
+#include "util/Buffer.hpp"
+#include "graphics/core/Model.hpp"
+
+/// See /doc/specs/vec3_model_spec.md
+namespace vec3 {
+    struct Material {
+        int flags;
+        std::string name;
+    };
+
+    struct Model {
+        std::string name;
+        model::Model model;
+        glm::vec3 origin;
+
+        Model& operator=(Model&&) = default;
+
+        ~Model();
+    };
+
+    struct File {
+        std::unordered_map<std::string, Model> models;
+        std::vector<Material> materials;
+
+        File(File&&) = default;
+
+        File& operator=(File&&) = default;
+    };
+
+    File load(const std::string_view file, const util::Buffer<ubyte>& src);
+}

src/files/files.cpp

@@ -75,7 +75,11 @@ std::unique_ptr<ubyte[]> files::read_bytes(
     const fs::path& filename, size_t& length
 ) {
     std::ifstream input(filename, std::ios::binary);
-    if (!input.is_open()) return nullptr;
+    if (!input.is_open()) {
+        throw std::runtime_error(
+            "could not to load file '" + filename.string() + "'"
+        );
+    }
     input.seekg(0, std::ios_base::end);
     length = input.tellg();
     input.seekg(0, std::ios_base::beg);
@@ -102,16 +106,11 @@ std::vector<ubyte> files::read_bytes(const fs::path& filename) {
 
 std::string files::read_string(const fs::path& filename) {
     size_t size;
-    std::unique_ptr<ubyte[]> bytes(read_bytes(filename, size));
-    if (bytes == nullptr) {
-        throw std::runtime_error(
-            "could not to load file '" + filename.string() + "'"
-        );
-    }
+    auto bytes = read_bytes(filename, size);
     return std::string((const char*)bytes.get(), size);
 }
 
-bool files::write_string(const fs::path& filename, const std::string content) {
+bool files::write_string(const fs::path& filename, std::string_view content) {
     std::ofstream file(filename);
     if (!file) {
         return false;

src/files/files.hpp

@@ -38,7 +38,7 @@ namespace files {
     uint append_bytes(const fs::path& file, const ubyte* data, size_t size);
 
     /// @brief Write string to the file
-    bool write_string(const fs::path& filename, const std::string content);
+    bool write_string(const fs::path& filename, std::string_view content);
 
     /// @brief Write dynamic data to the JSON file
     /// @param nice if true, human readable format will be used, otherwise

src/util/Buffer.hpp

@@ -37,6 +37,8 @@ namespace util {
 
         Buffer(std::nullptr_t) noexcept : ptr(nullptr), length(0) {}
 
+        Buffer& operator=(Buffer&&) = default;
+
         inline bool operator==(std::nullptr_t) const noexcept {
             return ptr == nullptr;
         }

test/coders/vec3.cpp

@@ -0,0 +1,12 @@
+#include <gtest/gtest.h>
+
+#include "coders/vec3.hpp"
+#include "files/files.hpp"
+
+TEST(VEC3, Decode) {
+    auto file = std::filesystem::u8path(
+        "../res/content/base/models/demo.vec3"
+    );
+    auto bytes = files::read_bytes_buffer(file);
+    auto model = vec3::load(file.u8string(), bytes);
+}