refactor(reactive): improve loaders

### Description

- Rename supported types
  - Before: `"cubeTexture" | "texture" | "gltfModel" | "video" | "audio"`
  - After: `"audio" | "image" | "video" | "gltf"`
- Now LoaderResource support is defined as `GLTF | ImageBitmap | ImageBitmap[] | AudioBuffer`
- Fix OrbitControl types

.changeset/metal-yaks-flash.md

@@ -0,0 +1,13 @@
+---
+"@quick-threejs/reactive": patch
+---
+
+# Logs
+
+## refactor(reactive): improve loaders
+
+- Rename supported types
+  - Before: `"cubeTexture" | "texture" | "gltfModel" | "video" | "audio"`
+  - After: `"audio" | "image" | "video" | "gltf"`
+- Now LoaderResource support is defined as `GLTF | ImageBitmap | ImageBitmap[] | AudioBuffer`
+- Fix OrbitControl types

packages/reactive/src/common/interfaces/loader.interface.ts

@@ -1,21 +1,15 @@
 import { JsonSerializable } from "threads";
-import { CubeTextureLoader, Texture, VideoTexture } from "three";
 import { GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";
 
 /** @description The sources of the {@link LoaderResource resources} to load. */
 export interface LoaderSource {
 	name: string;
-	type: "cubeTexture" | "texture" | "gltfModel" | "video" | "audio";
+	type: "audio" | "image" | "video" | "gltf";
 	path: string | string[];
 }
 
 /** @description Supported loadable resource. */
-export type LoaderResource =
-	| GLTF
-	| Texture
-	| CubeTextureLoader
-	| VideoTexture
-	| AudioBuffer;
+export type LoaderResource = GLTF | ImageBitmap | ImageBitmap[] | AudioBuffer;
 
 /** @description Represent a loaded resource. */
 export interface LoadedResourcePayload {

packages/reactive/src/core/app/debug/debug.service.ts

@@ -6,7 +6,7 @@ import {
 	GridHelper,
 	PerspectiveCamera
 } from "three";
-import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
+import { OrbitControls } from "three/examples/jsm/Addons.js";
 
 import { CameraService } from "../camera/camera.service";
 import { AppService } from "../app.service";

packages/reactive/src/core/app/loader/loader.controller.ts

@@ -29,7 +29,7 @@ export class LoaderController {
 		map((event) => {
 			const { payload } = event.data || {};
 
-			if (!!payload?.resource && payload.source.type === "gltfModel") {
+			if (payload?.source?.type === "gltf") {
 				const resource = payload.resource as unknown as {
 					animations?: AnimationClipJSON[];
 					cameras?: string[];
@@ -44,10 +44,10 @@ export class LoaderController {
 				const cameras = resource.cameras?.map(
 					(camera) => deserializeObject3D(camera) as Camera
 				);
-				const scene = deserializeObject3D(resource?.scene || "") as Group;
 				const scenes = resource.scenes?.map(
 					(scene) => deserializeObject3D(scene) as Group
 				);
+				const scene = scenes?.[0];
 
 				return {
 					...payload,
@@ -60,6 +60,8 @@ export class LoaderController {
 		share()
 	);
 	public readonly loadCompleted$ = this.load$.pipe(
-		filter((payload) => payload?.toLoadCount === payload.loadedCount)
+		filter(
+			(payload) => !!payload && payload.toLoadCount === payload.loadedCount
+		)
 	);
 }

packages/reactive/src/core/register/loader/loader.controller.ts

@@ -1,7 +1,7 @@
 import { serializeObject3D } from "@quick-threejs/utils";
 import { filter, map, Observable, share, Subject } from "rxjs";
 import { GLTF } from "three/examples/jsm/loaders/GLTFLoader";
-import { AnimationClipJSON, VideoTexture } from "three";
+import { AnimationClipJSON } from "three";
 import { inject, singleton } from "tsyringe";
 
 import {
@@ -37,6 +37,10 @@ export class LoaderController {
 				if ((payload?.resource as GLTF)?.parser) {
 					const _resource = payload.resource as GLTF;
 
+					const scenes = _resource.scenes.map((scene) =>
+						serializeObject3D(scene)
+					);
+
 					resource = {
 						animations: (payload?.resource as GLTF).animations.map(
 							// @ts-ignore <<toJSON methods doesn't require a parameter>>
@@ -46,14 +50,18 @@ export class LoaderController {
 							serializeObject3D(camera)
 						),
 						parser: { json: _resource.parser.json },
-						scene: serializeObject3D(_resource.scene),
-						scenes: _resource.scenes.map((scene) => serializeObject3D(scene)),
+						scene: scenes?.[0],
+						scenes,
 						userData: _resource.userData
 					};
 				}
 
-				if (payload?.resource instanceof VideoTexture)
-					resource = payload.resource.toJSON() as unknown as typeof resource;
+				if (
+					payload?.source?.type === "video" &&
+					Array.isArray(payload.resource)
+				) {
+					console.log("Serialized Texture ===>", payload.resource);
+				}
 
 				return {
 					...payload,

packages/reactive/src/core/register/loader/loader.module.ts

@@ -2,7 +2,6 @@ import "reflect-metadata";
 
 import { Subscription } from "rxjs";
 import { container, inject, singleton } from "tsyringe";
-import { CanvasTexture } from "three";
 
 import type {
 	Module,
@@ -30,7 +29,7 @@ export class LoaderModule implements Module {
 		);
 	}
 
-	private _performLoad(source: LoaderSource, resource?: LoaderResource) {
+	private _listenToLoad(source: LoaderSource, resource?: LoaderResource) {
 		this._controller.load$$.next({
 			source,
 			resource
@@ -46,39 +45,7 @@ export class LoaderModule implements Module {
 	}
 
 	public load() {
-		const firstSource = this._service.sources[0];
-		if (!firstSource) return;
-
-		this._performLoad(firstSource);
-
-		for (const source of this._service.sources) {
-			if (this._service.loadedResources[source.name]) return;
-
-			if (source.type === "gltfModel" && typeof source.path === "string")
-				this._service.loaders.gltfLoader?.load(source.path, (model) =>
-					this._performLoad(source, model)
-				);
-
-			if (source.type === "texture" && typeof source.path === "string")
-				this._service.loaders.textureLoader?.load(source.path, (texture) => {
-					this._performLoad(source, new CanvasTexture(texture));
-				});
-
-			if (source.type === "cubeTexture" && typeof source.path === "object")
-				this._service.loaders.cubeTextureLoader?.load(source.path, (texture) =>
-					this._performLoad(source, texture)
-				);
-
-			if (source.type === "video" && typeof source.path === "string")
-				this._service.loaders.videoLoader?.load(source.path, (texture) =>
-					this._performLoad(source, texture)
-				);
-
-			if (source.type === "audio" && typeof source.path === "string")
-				this._service.loaders.audioLoader?.load(source.path, (audioBuffer) => {
-					this._performLoad(source, audioBuffer);
-				});
-		}
+		this._service.load(this._listenToLoad.bind(this));
 	}
 
 	public getLoadedResources() {

packages/reactive/src/core/register/loader/loader.service.ts

@@ -1,11 +1,5 @@
 import { singleton } from "tsyringe";
-import {
-	AudioLoader,
-	CubeTextureLoader,
-	ImageBitmapLoader,
-	LoadingManager,
-	VideoTexture
-} from "three";
+import { AudioLoader, ImageBitmapLoader, LoadingManager } from "three";
 import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
 import { DRACOLoader } from "three/examples/jsm/loaders/DRACOLoader.js";
 
@@ -19,11 +13,10 @@ import {
 export class LoaderService {
 	public readonly loadingManager = new LoadingManager();
 	public readonly loaders: {
+		audioLoader?: AudioLoader;
 		dracoLoader?: DRACOLoader;
 		gltfLoader?: GLTFLoader;
-		textureLoader?: ImageBitmapLoader;
-		cubeTextureLoader?: CubeTextureLoader;
-		audioLoader?: AudioLoader;
+		imageLoader?: ImageBitmapLoader;
 		videoLoader?: LoaderService["videoLoader"];
 	} = {};
 
@@ -35,41 +28,77 @@ export class LoaderService {
 	/** @description The video loader. based on {@link HTMLVideoElement}. */
 	private get videoLoader() {
 		return {
-			load: (url: string, callback: (texture: VideoTexture) => unknown) => {
+			load: (url: string, callback: (texture: ImageBitmap[]) => unknown) => {
 				const element: HTMLVideoElement | undefined =
 					document.createElement("video");
 				element.muted = true;
 				element.loop = true;
+				element.crossOrigin = "anonymous";
 				element.controls = false;
 				element.playsInline = true;
 				element.src = url;
 				element.autoplay = true;
 
-				const oncanplaythrough = () => {
+				async function extractVideoFrames(video: HTMLVideoElement) {
+					const offscreenCanvas = new OffscreenCanvas(
+						video.videoWidth,
+						video.videoHeight
+					);
+					const ctx = offscreenCanvas.getContext("2d");
+					const frameRate = 30; // Target frame rate for extraction
+					const frames: ImageBitmap[] = [];
+
+					console.log("Extracting frames from video");
+
+					return new Promise<ImageBitmap[]>((resolve) => {
+						video.currentTime = 0;
+
+						while (video.currentTime < video.duration) {
+							ctx?.drawImage(
+								video,
+								0,
+								0,
+								offscreenCanvas.width,
+								offscreenCanvas.height
+							);
+							const bitmap = offscreenCanvas.transferToImageBitmap();
+							frames.push(bitmap);
+
+							if (video.currentTime < video.duration) {
+								video.currentTime += 1 / frameRate; // Move to the next frame
+							}
+						}
+						resolve(frames);
+
+						video.onerror = (error) => {
+							console.error("Error while extracting frames:", error);
+						};
+
+						// Start extraction
+						video.currentTime = 0;
+					});
+				}
+
+				const onLoadedData = async () => {
 					if (!element) return;
+					const frames = await extractVideoFrames(element);
+
+					console.log("Video loaded", frames);
 
-					element.play();
-					const texture = new VideoTexture(element);
-					const textureDispose = texture.dispose.bind(texture);
-					texture.dispose = () => {
-						(texture.image as HTMLVideoElement | undefined)?.remove();
-						textureDispose();
-					};
-					callback(texture);
+					callback(frames);
 
-					element.removeEventListener("canplaythrough", oncanplaythrough);
+					element.removeEventListener("loadeddata", onLoadedData);
 				};
-				element.addEventListener("canplaythrough", oncanplaythrough);
+				element.addEventListener("loadeddata", onLoadedData);
 			}
 		};
 	}
 
 	private _initLoaders() {
-		this.loaders.gltfLoader = new GLTFLoader(this.loadingManager);
-		this.loaders.textureLoader = new ImageBitmapLoader(this.loadingManager);
-		this.loaders.cubeTextureLoader = new CubeTextureLoader(this.loadingManager);
-		this.loaders.audioLoader = new AudioLoader(this.loadingManager);
 		this.loaders.dracoLoader = new DRACOLoader(this.loadingManager);
+		this.loaders.audioLoader = new AudioLoader(this.loadingManager);
+		this.loaders.gltfLoader = new GLTFLoader(this.loadingManager);
+		this.loaders.imageLoader = new ImageBitmapLoader(this.loadingManager);
 		this.loaders.videoLoader = this.videoLoader;
 	}
 
@@ -106,4 +135,38 @@ export class LoaderService {
 		this.loadedCount = loadedCount;
 		this.toLoadCount = toLoadCount;
 	}
+
+	public load(
+		onLoad?: (source: LoaderSource, resource?: LoaderResource) => unknown
+	) {
+		const firstSource = this.sources[0];
+		if (!firstSource) return;
+
+		onLoad?.(firstSource);
+
+		for (const source of this.sources) {
+			if (this.loadedResources[source.name] || typeof source.path !== "string")
+				return;
+
+			if (source.type === "gltf")
+				this.loaders.gltfLoader?.load(source.path, (model) =>
+					onLoad?.(source, model)
+				);
+
+			if (source.type === "audio")
+				this.loaders.audioLoader?.load(source.path, (audioBuffer) => {
+					onLoad?.(source, audioBuffer);
+				});
+
+			if (source.type === "image")
+				this.loaders.imageLoader?.load(source.path, (image) => {
+					onLoad?.(source, image);
+				});
+
+			if (source.type === "video")
+				this.loaders.videoLoader?.load(source.path, (texture) =>
+					onLoad?.(source, texture)
+				);
+		}
+	}
 }

samples/with-reactive/src/main.ts

@@ -2,6 +2,7 @@ import { register } from "@quick-threejs/reactive";
 import Stats from "stats.js";
 
 import chessPawn from "./assets/3D/pawn.glb?url";
+import matCapImg from "./assets/textures/matcap.jpg?url";
 
 import "./style.css";
 
@@ -17,12 +18,12 @@ register({
 		{
 			name: "pawn",
 			path: chessPawn,
-			type: "gltfModel"
+			type: "gltf"
 		},
 		{
-			name: "videoTexture",
-			path: "https://static.pexels.com/lib/videos/free-videos.mp4",
-			type: "video"
+			name: "matcap",
+			path: matCapImg,
+			type: "image"
 		}
 	],
 	onReady: async (app) => {