diff --git a/packages/dev/core/src/LibDeclarations/webxr.d.ts b/packages/dev/core/src/LibDeclarations/webxr.d.ts index bf8def809bd..3f72263c1a3 100644 --- a/packages/dev/core/src/LibDeclarations/webxr.d.ts +++ b/packages/dev/core/src/LibDeclarations/webxr.d.ts @@ -1200,6 +1200,21 @@ interface XRSession { enabledFeatures: string[]; } +// Raw camera access + +interface XRView { + readonly camera: XRCamera | undefined; +} + +interface XRCamera { + readonly width: number; + readonly height: number; +} + +interface XRWebGLBinding { + getCameraImage(camera: XRCamera): WebGLTexture | undefined; +} + /** * END: WebXR Depth Sensing Moudle * https://www.w3.org/TR/webxr-depth-sensing-1/ diff --git a/packages/dev/core/src/XR/features/WebXRAbstractFeature.ts b/packages/dev/core/src/XR/features/WebXRAbstractFeature.ts index 34919b5563f..13075dc488e 100644 --- a/packages/dev/core/src/XR/features/WebXRAbstractFeature.ts +++ b/packages/dev/core/src/XR/features/WebXRAbstractFeature.ts @@ -1,5 +1,6 @@ import type { IWebXRFeature } from "../webXRFeaturesManager"; -import type { Observer, Observable, EventState } from "../../Misc/observable"; +import type { Observer, EventState } from "../../Misc/observable"; +import { Observable } from "../../Misc/observable"; import type { Nullable } from "../../types"; import type { WebXRSessionManager } from "../webXRSessionManager"; @@ -30,6 +31,15 @@ export abstract class WebXRAbstractFeature implements IWebXRFeature { */ public xrNativeFeatureName: string = ""; + /** + * Observers registered here will be executed when the feature is attached + */ + public onFeatureAttachObservable: Observable = new Observable(); + /** + * Observers registered here will be executed when the feature is detached + */ + public onFeatureDetachObservable: Observable = new Observable(); + /** * Construct a new (abstract) WebXR feature * @param _xrSessionManager the xr session manager for this feature @@ -67,6 +77,7 @@ export abstract class WebXRAbstractFeature implements IWebXRFeature { this._attached = true; this._addNewAttachObserver(this._xrSessionManager.onXRFrameObservable, (frame) => this._onXRFrame(frame)); + this.onFeatureAttachObservable.notifyObservers(this); return true; } @@ -84,6 +95,7 @@ export abstract class WebXRAbstractFeature implements IWebXRFeature { this._removeOnDetach.forEach((toRemove) => { toRemove.observable.remove(toRemove.observer); }); + this.onFeatureDetachObservable.notifyObservers(this); return true; } @@ -93,6 +105,8 @@ export abstract class WebXRAbstractFeature implements IWebXRFeature { public dispose(): void { this.detach(); this.isDisposed = true; + this.onFeatureAttachObservable.clear(); + this.onFeatureDetachObservable.clear(); } /** diff --git a/packages/dev/core/src/XR/features/WebXRRawCameraAccess.ts b/packages/dev/core/src/XR/features/WebXRRawCameraAccess.ts new file mode 100644 index 00000000000..8b181360676 --- /dev/null +++ b/packages/dev/core/src/XR/features/WebXRRawCameraAccess.ts @@ -0,0 +1,217 @@ +import { WebXRFeatureName, WebXRFeaturesManager } from "../webXRFeaturesManager"; +import type { WebXRSessionManager } from "../webXRSessionManager"; +import { WebXRAbstractFeature } from "./WebXRAbstractFeature"; +import { Observable } from "../../Misc/observable"; +import { Constants } from "../../Engines/constants"; +import { WebGLHardwareTexture } from "../../Engines/WebGL/webGLHardwareTexture"; +import { InternalTexture, InternalTextureSource } from "../../Materials/Textures/internalTexture"; +import { BaseTexture } from "core/Materials/Textures/baseTexture"; + +/** + * Options for raw camera access + */ +export interface IWebXRRawCameraAccessOptions { + /** + * Keep the created textures and metadata when detaching the feature. + */ + doNotDisposeOnDetach?: boolean; +} + +/** + * WebXR Feature for WebXR raw camera access + * @since + * @see https://immersive-web.github.io/raw-camera-access/ + */ +export class WebXRRawCameraAccess extends WebXRAbstractFeature { + private _cachedInternalTextures: InternalTexture[] = []; + /** + * This is an array of camera views + * Note that mostly the array will contain a single view + * If you want to know the order of the views, use the `viewIndex` array + */ + public texturesData: BaseTexture[] = []; + /** + * If needed, this array will contain the eye definition of each texture in `texturesArray` + */ + public viewIndex: string[] = []; + + /** + * If needed, this array will contain the camera's intrinsics + * You can use this data to convert from camera space to screen space and vice versa + */ + public cameraIntrinsics: { + u0: number; + v0: number; + ax: number; + ay: number; + gamma: number; + width: number; + height: number; + viewportX: number; + viewportY: number; + }[] = []; + + /** + * An observable that will notify when the camera's textures are updated + */ + public onTexturesUpdatedObservable: Observable = new Observable(); + + private _glBinding?: XRWebGLBinding; + private _glContext: WebGLRenderingContext; + + /** + * The module's name + */ + public static readonly Name = WebXRFeatureName.RAW_CAMERA_ACCESS; + + /** + * The (Babylon) version of this module. + * This is an integer representing the implementation version. + * This number does not correspond to the WebXR specs version + */ + public static readonly Version = 1; + + /** + * Creates a new instance of the feature + * @param _xrSessionManager the WebXRSessionManager + * @param options options for the Feature + */ + constructor( + _xrSessionManager: WebXRSessionManager, + public readonly options: IWebXRRawCameraAccessOptions = {} + ) { + super(_xrSessionManager); + this.xrNativeFeatureName = "camera-access"; + } + + public attach(force?: boolean | undefined): boolean { + if (!super.attach(force)) { + return false; + } + + this._glContext = this._xrSessionManager.scene.getEngine()._gl; + this._glBinding = new XRWebGLBinding(this._xrSessionManager.session, this._glContext); + + return true; + } + + public detach(): boolean { + if (!super.detach()) { + return false; + } + this._glBinding = undefined; + if (!this.options.doNotDisposeOnDetach) { + this._cachedInternalTextures.forEach((t) => t.dispose()); + this.texturesData.forEach((t) => t.dispose()); + this._cachedInternalTextures.length = 0; + this.texturesData.length = 0; + this.cameraIntrinsics.length = 0; + } + return true; + } + + /** + * Dispose this feature and all of the resources attached + */ + public dispose(): void { + super.dispose(); + this.onTexturesUpdatedObservable.clear(); + } + + /** + * @see https://github.com/immersive-web/raw-camera-access/blob/main/explainer.md + */ + private _updateCameraIntrinsics(view: XRView, index: number): void { + const cameraViewport = { + width: view.camera!.width, + height: view.camera!.height, + x: 0, + y: 0, + }; + const p = view.projectionMatrix; + + // Principal point in pixels (typically at or near the center of the viewport) + const u0 = ((1 - p[8]) * cameraViewport.width) / 2 + cameraViewport.x; + const v0 = ((1 - p[9]) * cameraViewport.height) / 2 + cameraViewport.y; + + // Focal lengths in pixels (these are equal for square pixels) + const ax = (cameraViewport.width / 2) * p[0]; + const ay = (cameraViewport.height / 2) * p[5]; + + // Skew factor in pixels (nonzero for rhomboid pixels) + const gamma = (cameraViewport.width / 2) * p[4]; + this.cameraIntrinsics[index] = { + u0, + v0, + ax, + ay, + gamma, + width: cameraViewport.width, + height: cameraViewport.height, + viewportX: cameraViewport.x, + viewportY: cameraViewport.y, + }; + } + + private _updateInternalTextures(view: XRView, index = 0): boolean { + if (!view.camera) { + return false; + } + this.viewIndex[index] = view.eye; + const lp = this._glBinding?.getCameraImage(view.camera); + + if (!this._cachedInternalTextures[index]) { + const internalTexture = new InternalTexture(this._xrSessionManager.scene.getEngine(), InternalTextureSource.Unknown, true); + internalTexture.isCube = true; + internalTexture.invertY = false; + // internalTexture._useSRGBBuffer = this.options.reflectionFormat === "srgba8"; + internalTexture.format = Constants.TEXTUREFORMAT_RGBA; + internalTexture.generateMipMaps = true; + internalTexture.type = Constants.TEXTURETYPE_FLOAT; + internalTexture.samplingMode = Constants.TEXTURE_LINEAR_LINEAR_MIPLINEAR; + internalTexture.width = view.camera.width; + internalTexture.height = view.camera.height; + internalTexture._cachedWrapU = Constants.TEXTURE_WRAP_ADDRESSMODE; + internalTexture._cachedWrapV = Constants.TEXTURE_WRAP_ADDRESSMODE; + internalTexture._hardwareTexture = new WebGLHardwareTexture(lp, this._glContext); + this._cachedInternalTextures[index] = internalTexture; + // create the base texture + const texture = new BaseTexture(this._xrSessionManager.scene); + texture.name = `WebXR Raw Camera Access (${index})`; + texture._texture = this._cachedInternalTextures[index]; + this.texturesData[index] = texture; + // get the camera intrinsics + this._updateCameraIntrinsics(view, index); + } else { + // make sure the webgl texture is updated. Should happen automatically + this._cachedInternalTextures[index]._hardwareTexture?.set(lp); + } + this._cachedInternalTextures[index].isReady = true; + return true; + } + + protected _onXRFrame(_xrFrame: XRFrame): void { + const referenceSPace = this._xrSessionManager.referenceSpace; + + const pose = _xrFrame.getViewerPose(referenceSPace); + if (!pose || !pose.views) { + return; + } + let updated = true; + pose.views.forEach((view, index) => { + updated = updated && this._updateInternalTextures(view, index); + }); + if (updated) { + this.onTexturesUpdatedObservable.notifyObservers(this.texturesData); + } + } +} + +WebXRFeaturesManager.AddWebXRFeature( + WebXRRawCameraAccess.Name, + (xrSessionManager, options) => { + return () => new WebXRRawCameraAccess(xrSessionManager, options); + }, + WebXRRawCameraAccess.Version, + false +); diff --git a/packages/dev/core/src/XR/features/index.ts b/packages/dev/core/src/XR/features/index.ts index 5db234ef8e2..de55538bf24 100644 --- a/packages/dev/core/src/XR/features/index.ts +++ b/packages/dev/core/src/XR/features/index.ts @@ -20,3 +20,4 @@ export * from "./WebXRWalkingLocomotion"; export * from "./WebXRLayers"; export * from "./WebXRDepthSensing"; export * from "./WebXRSpaceWarp"; +export * from "./WebXRRawCameraAccess"; diff --git a/packages/dev/core/src/XR/webXRFeaturesManager.ts b/packages/dev/core/src/XR/webXRFeaturesManager.ts index 336cda466cc..697e9c33dfa 100644 --- a/packages/dev/core/src/XR/webXRFeaturesManager.ts +++ b/packages/dev/core/src/XR/webXRFeaturesManager.ts @@ -1,6 +1,7 @@ import type { WebXRSessionManager } from "./webXRSessionManager"; import type { IDisposable } from "../scene"; import { Tools } from "../Misc/tools"; +import type { Observable } from "core/Misc/observable"; /** * Defining the interface required for a (webxr) feature @@ -58,6 +59,15 @@ export interface IWebXRFeature extends IDisposable { * If this feature requires to extend the XRSessionInit object, this function will return the partial XR session init object */ getXRSessionInitExtension?: () => Promise>; + + /** + * Triggered when the feature is attached + */ + onFeatureAttachObservable: Observable; + /** + * Triggered when the feature is detached + */ + onFeatureDetachObservable: Observable; } /** @@ -144,6 +154,10 @@ export class WebXRFeatureName { * The name of the WebXR Space Warp feature */ public static readonly SPACE_WARP = "xr-space-warp"; + /** + * The name of the WebXR Raw Camera Access feature + */ + public static readonly RAW_CAMERA_ACCESS = "xr-raw-camera-access"; } /**