gh-84: add async file io system & use for assets files input

engine/runtime/asset/asset_library_fs.cpp

@@ -29,6 +29,7 @@
 
 #include "asset/asset_manager.hpp"
 #include "core/string_utils.hpp"
+#include "io/async_file_system.hpp"
 #include "io/tree.hpp"
 #include "io/tree_yaml.hpp"
 #include "platform/file_system.hpp"
@@ -39,9 +40,10 @@
 namespace wmoge {
 
     AssetLibraryFileSystem::AssetLibraryFileSystem(std::string directory, IocContainer* ioc) {
-        m_directory    = std::move(directory);
-        m_file_system  = ioc->resolve_value<FileSystem>();
-        m_rtti_storage = ioc->resolve_value<RttiTypeStorage>();
+        m_directory         = std::move(directory);
+        m_file_system       = ioc->resolve_value<FileSystem>();
+        m_async_file_system = ioc->resolve_value<IoAsyncFileSystem>();
+        m_rtti_storage      = ioc->resolve_value<RttiTypeStorage>();
     }
 
     std::string AssetLibraryFileSystem::get_name() const {
@@ -53,6 +55,7 @@ namespace wmoge {
     }
 
     bool AssetLibraryFileSystem::has_asset(const AssetId& name) {
+        WG_AUTO_PROFILE_ASSET("AssetLibraryFileSystem::has_asset");
         return m_file_system->exists(make_asset_meta_path(m_directory, name, m_asset_ext));
     }
 
@@ -69,34 +72,25 @@ namespace wmoge {
 
         return WG_OK;
     }
+
+    static std::string make_asset_data_path(const std::string& directory, const AssetId& name) {
+        return directory + name.str();
+    }
+
     Status AssetLibraryFileSystem::find_asset_data_meta(const Strid& name, AssetDataMeta& meta) {
         WG_AUTO_PROFILE_ASSET("AssetLibraryFileSystem::find_asset_data_meta");
 
-        const std::string path = m_directory + name.str();
-
-        WG_CHECKED(m_file_system->get_file_size(path, meta.size));
+        WG_CHECKED(m_file_system->get_file_size(make_asset_data_path(m_directory, name), meta.size));
         meta.size_compressed = 0;
         meta.hash            = Sha256();
         meta.compression     = AssetCompressionMode::None;
 
         return WG_OK;
     }
+
     Async AssetLibraryFileSystem::read_data(const Strid& name, array_view<std::uint8_t> data) {
         WG_AUTO_PROFILE_ASSET("AssetLibraryFileSystem::read_data");
-
-        const std::string path = m_directory + name.str();
-        FileOpenModeFlags mode = {FileOpenMode::In, FileOpenMode::Binary};
-        Ref<File>         file;
-
-        if (!m_file_system->open_file(path, file, mode)) {
-            return Async::failed();
-        }
-
-        if (!file->nread(data.data(), data.size())) {
-            return Async::failed();
-        }
-
-        return Async::completed();
+        return m_async_file_system->read_file(make_asset_data_path(m_directory, name), data).as_async();
     }
 
 }// namespace wmoge

engine/runtime/asset/asset_library_fs.hpp

@@ -47,10 +47,11 @@ namespace wmoge {
         Async       read_data(const Strid& name, array_view<std::uint8_t> data) override;
 
     private:
-        class FileSystem*      m_file_system;
-        class RttiTypeStorage* m_rtti_storage;
-        std::string            m_directory;
-        std::string            m_asset_ext = ".asset";
+        class FileSystem*        m_file_system;
+        class IoAsyncFileSystem* m_async_file_system;
+        class RttiTypeStorage*   m_rtti_storage;
+        std::string              m_directory;
+        std::string              m_asset_ext = ".asset";
     };
 
 }// namespace wmoge

engine/runtime/asset/asset_loader.hpp

@@ -65,9 +65,8 @@ namespace wmoge {
      * @brief Context passed to the loader
      */
     struct AssetLoadContext {
-        IoContext     io_context;
-        AssetMeta     asset_meta;
-        AssetLibrary* asset_library = nullptr;
+        IoContext io_context;
+        AssetMeta asset_meta;
     };
 
     /**

engine/runtime/asset/asset_manager.cpp

@@ -58,7 +58,6 @@ namespace wmoge {
 
         std::lock_guard guard(m_mutex);
 
-        // already loaded and cached
         Ref<Asset> res = find(name);
         if (res) {
             std::optional<Ref<Asset>> res_opt(res);
@@ -67,122 +66,113 @@ namespace wmoge {
             return AsyncResult<Ref<Asset>>(async_op);
         }
 
-        // not yet cached, but is loading
         auto loading = m_loading.find(name);
         if (loading != m_loading.end()) {
-            auto& async_op = loading->second.async_op;
+            auto& async_op = loading->second->async_op;
             return AsyncResult<Ref<Asset>>(async_op);
         }
 
-        // try to find asset library
         std::optional<AssetLibrary*> asset_library = resolve_asset(name);
         if (!asset_library) {
             WG_LOG_ERROR("failed to resolve asset library for " << name);
             return AsyncResult<Ref<Asset>>::failed();
         }
 
-        // try to find resolve meta info
         AssetMeta asset_meta;
         if (!asset_library.value()->find_asset_meta(name, asset_meta)) {
             WG_LOG_ERROR("failed to find meta info for " << name);
             return AsyncResult<Ref<Asset>>::failed();
         }
 
-        // try find loader
         std::optional<AssetLoader*> loader = find_loader(asset_meta.loader);
         if (!loader) {
             WG_LOG_ERROR("failed to find loader for " << name);
             return AsyncResult<Ref<Asset>>::failed();
         }
 
-        // get dependencies which still loading or already loaded
         buffered_vector<Async> deps;
         for (const Strid& dep : asset_meta.deps) {
             deps.push_back(load_async(dep).as_async());
         }
 
-        // create loading state to track result
-        AsyncOp<Ref<Asset>> async_op = make_async_op<Ref<Asset>>();
-
-        // create task to load
-        Task task(name, [=, meta = std::move(asset_meta), library = asset_library.value(), loader = loader.value()](TaskContext&) {
-            Timer timer;
-            timer.start();
-
-            Ref<Asset>             asset;
-            AssetLoadContext       context;
-            AssetLoadRequest       request;
-            AssetLoadResult        result;
-            std::vector<Ref<Data>> data_buffers;
-
-            context.io_context.add<IocContainer*>(m_ioc_container);
-            context.asset_meta = std::move(meta);
-
-            if (!loader->fill_request(context, name, request)) {
-                WG_LOG_ERROR("failed fill_request for " << name);
+        Ref<LoadState> load_state      = make_ref<LoadState>();
+        load_state->deps               = std::move(deps);
+        load_state->async_op           = make_async_op<Ref<Asset>>();
+        load_state->library            = asset_library.value();
+        load_state->loader             = loader.value();
+        load_state->context.asset_meta = std::move(asset_meta);
+        load_state->context.io_context.add<IocContainer*>(m_ioc_container);
+
+        Task task_fill_requests(name, [=](TaskContext&) -> int {
+            if (!load_state->loader->fill_request(load_state->context, name, load_state->request)) {
+                WG_LOG_ERROR("failed to fill request for " << name);
                 return 1;
             }
 
-            data_buffers.reserve(request.data_files.size());
-            for (const auto& tag_path : request.data_files) {
-                AssetDataMeta data_meta;
+            buffered_vector<Async> file_data_requests;
+
+            file_data_requests.reserve(load_state->request.data_files.size());
+            load_state->data_buffers.reserve(load_state->request.data_files.size());
 
-                if (!library->find_asset_data_meta(tag_path.second, data_meta)) {
+            for (const auto& tag_path : load_state->request.data_files) {
+                AssetDataMeta data_meta;
+                if (!load_state->library->find_asset_data_meta(tag_path.second, data_meta)) {
                     WG_LOG_ERROR("failed to find meta for " << tag_path.second);
                     return 1;
                 }
 
-                Ref<Data>& buffer = data_buffers.emplace_back();
+                Ref<Data>& buffer = load_state->data_buffers.emplace_back();
                 buffer            = make_ref<Data>(data_meta.size);
 
-                auto async_result = library->read_data(tag_path.second, {buffer->buffer(), buffer->size()});
-                async_result.wait_completed();
+                auto async_result = load_state->library->read_data(tag_path.second, {buffer->buffer(), buffer->size()});
+                file_data_requests.push_back(async_result);
+
+                load_state->result.add_data_file(tag_path.first, {buffer->buffer(), buffer->size()});
+            }
 
-                if (async_result.is_failed()) {
-                    WG_LOG_ERROR("failed to read " << tag_path.second);
+            Task task_load(name, [=](TaskContext&) -> int {
+                Timer timer;
+                timer.start();
+
+                Ref<Asset> asset;
+                if (!load_state->loader->load(load_state->context, name, load_state->result, asset)) {
+                    WG_LOG_ERROR("failed load for " << name);
                     return 1;
                 }
 
-                result.add_data_file(tag_path.first, {buffer->buffer(), buffer->size()});
-            }
+                timer.stop();
+                WG_LOG_INFO("load asset " << name << ", time: " << timer.get_elapsed_sec() << " sec");
 
-            if (!loader->load(context, name, result, asset)) {
-                WG_LOG_ERROR("failed load for " << name);
-                return 1;
-            }
+                std::lock_guard guard(m_mutex);
+                asset->set_id(name);
+                asset->set_release_callback(m_callback);
+                m_assets[name] = WeakRef<Asset>(asset);
+                load_state->async_op->set_result(std::move(asset));
 
-            timer.stop();
-            WG_LOG_INFO("load asset " << name << ", time: " << timer.get_elapsed_sec() << " sec");
+                return 0;
+            });
 
-            if (asset->get_name().empty()) {
-                asset->set_id(name);
-            }
+            task_load.schedule(Async::join(array_view(file_data_requests))).add_on_completion([=](AsyncStatus status, std::optional<int>&) {
+                if (status == AsyncStatus::Failed) {
+                    load_state->async_op->set_failed();
+                }
+                std::lock_guard guard(m_mutex);
+                m_loading.erase(name);
+            });
 
-            std::lock_guard guard(m_mutex);
-            asset->set_release_callback(m_callback);
-            m_assets[name] = WeakRef<Asset>(asset);
-            async_op->set_result(std::move(asset));
             return 0;
         });
 
-        // schedule to run only if all deps are loaded
-        auto task_hnd = task.schedule(Async::join(array_view(deps)));
-
-        // add erase of loading state here, since it is possible, that task can be aborted
-        task_hnd.add_on_completion([this, name, async_op](AsyncStatus status, std::optional<int>&) {
-            std::lock_guard guard(m_mutex);
+        task_fill_requests.schedule(Async::join(array_view(deps))).add_on_completion([=](AsyncStatus status, std::optional<int>&) {
             if (status == AsyncStatus::Failed) {
-                async_op->set_failed();
+                load_state->async_op->set_failed();
             }
+            std::lock_guard guard(m_mutex);
             m_loading.erase(name);
         });
 
-        auto& state    = m_loading[name];
-        state.deps     = std::move(deps);
-        state.task_hnd = std::move(task_hnd);
-        state.async_op = std::move(async_op);
-
-        return AsyncResult<Ref<Asset>>(state.async_op);
+        m_loading[name] = load_state;
+        return AsyncResult<Ref<Asset>>(load_state->async_op);
     }
 
     Ref<Asset> AssetManager::load(const AssetId& name) {

engine/runtime/asset/asset_manager.hpp

@@ -78,15 +78,20 @@ namespace wmoge {
         void                         load_loaders();
 
     private:
-        struct LoadState {
+        struct LoadState : public RefCnt {
             buffered_vector<Async> deps;
             AsyncOp<Ref<Asset>>    async_op;
-            TaskHnd                task_hnd;
+            AssetLoadContext       context;
+            AssetLoadRequest       request;
+            AssetLoadResult        result;
+            AssetLibrary*          library = nullptr;
+            AssetLoader*           loader  = nullptr;
+            std::vector<Ref<Data>> data_buffers;
         };
 
         std::vector<std::shared_ptr<AssetLibrary>>   m_libraries;
         flat_map<AssetId, WeakRef<Asset>>            m_assets;
-        flat_map<AssetId, LoadState>                 m_loading;
+        flat_map<AssetId, Ref<LoadState>>            m_loading;
         flat_map<Strid, Ref<AssetLoader>>            m_loaders;
         std::shared_ptr<std::function<void(Asset*)>> m_callback;
 

engine/runtime/core/array_view.hpp

@@ -66,8 +66,7 @@ namespace wmoge {
         }
 
         [[nodiscard]] std::size_t size() const { return m_size; }
-        [[nodiscard]] T*          data() { return m_data; }
-        [[nodiscard]] const T*    data() const { return m_data; }
+        [[nodiscard]] T*          data() const { return m_data; }
         [[nodiscard]] bool        empty() const { return !m_size; }
 
         const T* begin() const { return m_data; }

engine/runtime/engine.hpp

@@ -124,6 +124,7 @@
 #include "grc/texture_manager.hpp"
 #include "grc/texture_resize.hpp"
 
+#include "io/async_file_system.hpp"
 #include "io/base64.hpp"
 #include "io/compression.hpp"
 #include "io/context.hpp"

engine/runtime/glsl/glsl_shader_compiler.cpp

@@ -368,7 +368,7 @@ namespace wmoge {
     GlslShaderCompilerAdapter::GlslShaderCompilerAdapter(GfxShaderPlatform platform) {
         m_platform      = platform;
         m_glsl_compiler = IocContainer::iresolve_v<GlslShaderCompiler>();
-        m_task_manager  = IocContainer::iresolve_v<ShaderCompilerTaskManager>();
+        m_task_manager  = IocContainer::iresolve_v<ShaderTaskManager>();
     }
 
     Async GlslShaderCompilerAdapter::compile(const Ref<ShaderCompilerRequest>& request, const Async& depends_on) {

engine/runtime/grc/pso_cache.cpp

@@ -44,7 +44,7 @@ namespace wmoge {
 
     PsoCache::PsoCache() {
         m_shader_library = IocContainer::iresolve_v<ShaderLibrary>();
-        m_task_manager   = IocContainer::iresolve_v<ShaderCompilerTaskManager>();
+        m_task_manager   = IocContainer::iresolve_v<ShaderTaskManager>();
         m_gfx_driver     = IocContainer::iresolve_v<GfxDriver>();
     }
 

engine/runtime/grc/shader_compiler.hpp

@@ -196,13 +196,13 @@ namespace wmoge {
     WG_RTTI_END;
 
     /**
-     * @class ShaderCompilerTaskManager
+     * @class ShaderTaskManager
      * @brief Task manager to schedule shader compilation jobs
     */
-    class ShaderCompilerTaskManager : public TaskManager {
+    class ShaderTaskManager : public TaskManager {
     public:
-        ShaderCompilerTaskManager(int num_workers) : TaskManager(num_workers, "shader-compiler") {}
-        ~ShaderCompilerTaskManager() = default;
+        ShaderTaskManager(int num_workers) : TaskManager(num_workers, "shader-system") {}
+        ~ShaderTaskManager() = default;
     };
 
 }// namespace wmoge

engine/runtime/grc/shader_manager.cpp

@@ -56,7 +56,7 @@ namespace wmoge {
     ShaderManager::ShaderManager() {
         WG_AUTO_PROFILE_GRC("ShaderManager::ShaderManager");
 
-        m_task_manager    = IocContainer::iresolve_v<ShaderCompilerTaskManager>();
+        m_task_manager    = IocContainer::iresolve_v<ShaderTaskManager>();
         m_file_system     = IocContainer::iresolve_v<FileSystem>();
         m_gfx_driver      = IocContainer::iresolve_v<GfxDriver>();
         m_texture_manager = IocContainer::iresolve_v<TextureManager>();