From c1fe13b6ed35ded82add967042191b576584fbfb Mon Sep 17 00:00:00 2001
From: Tom Birdsong <tom.birdsong@kitware.com>
Date: Tue, 26 Sep 2023 15:26:54 -0400
Subject: [PATCH 1/3] feat(WebXR): Add initial support for WebXR viewing with
 VTK.js

Adds initial support for viewing VolView scene in WebXR on the local
device. Note that performance may vary widely across devices.
---
 src/components/RenderingModule.vue |  15 +++-
 src/components/WebXRRendering.vue  | 109 +++++++++++++++++++++++++++++
 src/store/view-configs/webxr.ts    |  22 ++++++
 3 files changed, 145 insertions(+), 1 deletion(-)
 create mode 100644 src/components/WebXRRendering.vue
 create mode 100644 src/store/view-configs/webxr.ts

diff --git a/src/components/RenderingModule.vue b/src/components/RenderingModule.vue
index 258fd6d2e..a47c21a3d 100644
--- a/src/components/RenderingModule.vue
+++ b/src/components/RenderingModule.vue
@@ -4,10 +4,11 @@ import { useCurrentImage } from '../composables/useCurrentImage';
 import VolumeProperties from './VolumeProperties.vue';
 import VolumeRendering from './VolumeRendering.vue';
 import VolumePresets from './VolumePresets.vue';
+import WebXRRendering from './WebXRRendering.vue';
 import LayerList from './LayerList.vue';
 
 export default defineComponent({
-  components: { VolumeRendering, VolumePresets, VolumeProperties, LayerList },
+  components: { VolumeRendering, VolumePresets, VolumeProperties, WebXRRendering, LayerList },
   setup() {
     const { currentImageData } = useCurrentImage();
     const hasCurrentImage = computed(() => !!currentImageData.value);
@@ -31,6 +32,7 @@ export default defineComponent({
     <template v-if="hasCurrentImage">
       <volume-rendering />
       <v-expansion-panels v-model="panels" multiple variant="accordion">
+
         <v-expansion-panel value="preset">
           <v-expansion-panel-title>
             <v-icon class="flex-grow-0 mr-4">mdi-palette</v-icon>
@@ -60,6 +62,17 @@ export default defineComponent({
             <layer-list />
           </v-expansion-panel-text>
         </v-expansion-panel>
+        
+        <v-expansion-panel value="webxr">
+          <v-expansion-panel-title>
+            <v-icon class="flex-grow-0 mr-4">mdi-palette</v-icon>
+            WebXR Rendering
+          </v-expansion-panel-title>
+          <v-expansion-panel-text>
+            <web-x-r-rendering />
+          </v-expansion-panel-text>
+        </v-expansion-panel>
+        
       </v-expansion-panels>
     </template>
     <template v-else>
diff --git a/src/components/WebXRRendering.vue b/src/components/WebXRRendering.vue
new file mode 100644
index 000000000..5a1eecf7f
--- /dev/null
+++ b/src/components/WebXRRendering.vue
@@ -0,0 +1,109 @@
+<script setup lang="ts">
+import {
+  ref,
+} from 'vue';
+
+import { XrSessionTypes } from '@kitware/vtk.js/Rendering/WebXR/RenderWindowHelper/Constants.js';
+import { onVTKEvent } from '@/src/composables/onVTKEvent';
+import { useViewProxy } from '../composables/useViewProxy';
+import useVTKWebXRStore from '../store/view-configs/webxr';
+import { ViewProxyType } from '../core/proxies';
+import { InitViewIDs } from '../config';
+
+const TARGET_VIEW_ID = InitViewIDs.Three;
+const vtkWebXRStore = useVTKWebXRStore();
+
+const xrSessionRunning = ref(vtkWebXRStore.isXrSessionRunning());
+    
+const namedXrSessionTypes = [
+  { value: 0, title: 'VR Headset'},
+  { value: 1, title: 'AR Mobile Device'},
+  { value: 2, title: 'Holographic Display'},
+  { value: 3, title: 'AR Headset'},
+];
+
+function updateXRStateFunc() {
+  xrSessionRunning.value = !!vtkWebXRStore.xrHelper.getXrSession();
+};
+
+function startXRFunc(sessionType:XrSessionTypes) {
+  const { viewProxy } = useViewProxy(TARGET_VIEW_ID, ViewProxyType.Volume);
+  vtkWebXRStore.xrHelper.setRenderWindow(viewProxy.value.getOpenGLRenderWindow());
+  vtkWebXRStore.xrHelper.startXR(sessionType);
+}
+
+onVTKEvent(vtkWebXRStore.xrHelper, 'onModified', updateXRStateFunc);
+
+function onSendToXRClick() {
+  if(vtkWebXRStore.xrHelper.getXrSession() !== null) {
+    throw new Error('Cannot send to XR: session already running');
+  }
+  if(vtkWebXRStore.selectedXrSessionType === null) {
+    throw new Error('Cannot send to XR: no session type selected');
+  }
+
+  const xrSessionType = vtkWebXRStore.selectedXrSessionType ? vtkWebXRStore.selectedXrSessionType.value : 0;
+
+  if(xrSessionType === XrSessionTypes.LookingGlassVR) {
+    // The Looking Glass WebXR Polyfill overrides support for other XR sessions.
+    vtkWebXRStore.isXrSessionTypeLocked = true;
+
+    // Import the Looking Glass WebXR Polyfill override.
+    // Assumes that the Looking Glass Bridge native application is already running.
+    // See https://docs.lookingglassfactory.com/developer-tools/webxr
+    import(
+      // @ts-expect-error TS2307
+      // eslint-disable-next-line import/no-unresolved, import/extensions
+      /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js'
+    ).then((obj) => {
+      // eslint-disable-next-line no-new
+      new obj.LookingGlassWebXRPolyfill();
+
+      // There is a delay between when the Looking Glass polyfill starts and when
+      // the Looking Glass Bridge application finishes populating device information.
+      // Attempting to start the XR session in the intermediate period can result in
+      // undefined behavior such as a failure to start the session or an attempt to start
+      // a standard VR HMD session.
+      // A timeout of 1 second is used to account for this delay as a workaround.
+      // In the future we may explore handling an event notification from the Looking Glass polyfill
+      // when a new Looking Glass device is ready for WebXR rendering.
+      setTimeout(() => startXRFunc(xrSessionType), 1000);
+    });
+  } else if((navigator as any).xr === undefined) {
+    import(
+    // @ts-expect-error TS2307
+    // eslint-disable-next-line import/no-unresolved, import/extensions
+    /* webpackIgnore: true */ 'https://cdn.jsdelivr.net/npm/webxr-polyfill@latest/build/webxr-polyfill.js'
+    ).then(() => {
+      startXRFunc(xrSessionType);
+    })
+  } else {
+    startXRFunc(xrSessionType);
+  }
+};
+
+function onReturnFromXRClick() {
+  if(vtkWebXRStore.xrHelper.getXrSession() === null) {
+    throw new Error('Cannot return from XR: no session running');
+  }
+  vtkWebXRStore.xrHelper.stopXR();
+};
+
+</script>
+
+<template>
+  <div class="mx-2">
+    <v-combobox
+    v-model="vtkWebXRStore.selectedXrSessionType"
+    :items="namedXrSessionTypes"
+    label="Mixed Reality Session Type"
+    item-text="title"
+    item-value="value"
+    :disabled="vtkWebXRStore.isXrSessionTypeLocked"
+    />
+    <button v-if="!xrSessionRunning && vtkWebXRStore.selectedXrSessionType" @click="onSendToXRClick">
+      Send to {{ vtkWebXRStore.selectedXrSessionType.title }}
+    </button>
+    <button v-if="xrSessionRunning" @click="onReturnFromXRClick">Return from XR</button>
+  </div>
+</template>
diff --git a/src/store/view-configs/webxr.ts b/src/store/view-configs/webxr.ts
new file mode 100644
index 000000000..0e9905b70
--- /dev/null
+++ b/src/store/view-configs/webxr.ts
@@ -0,0 +1,22 @@
+import { defineStore } from 'pinia';
+import { ref, Ref } from 'vue';
+import vtkWebXRRenderWindowHelper from '@kitware/vtk.js/Rendering/WebXR/RenderWindowHelper';
+
+export const useVTKWebXRStore = defineStore('vtkWebXR', () => {
+    const xrHelper = vtkWebXRRenderWindowHelper.newInstance();
+    const selectedXrSessionType : Ref<{title: string, value: number} | null> = ref(null);
+    const isXrSessionTypeLocked = ref(false);
+
+    function isXrSessionRunning() : Boolean {
+        return !!xrHelper.getXrSession();
+    }
+
+  return {
+    xrHelper,
+    selectedXrSessionType,
+    isXrSessionTypeLocked,
+    isXrSessionRunning
+  };
+});
+
+export default useVTKWebXRStore;

From 9c02c1f596150952dbc7dbaf4fe102bab388a075 Mon Sep 17 00:00:00 2001
From: Tom Birdsong <tom.birdsong@kitware.com>
Date: Thu, 26 Oct 2023 16:35:42 -0400
Subject: [PATCH 2/3] WIP: Update `package` and `package-lock` for VTK.js@v29

---
 package-lock.json | 56 +++++++++++++++++++++++++++++++++++++++++------
 package.json      |  2 +-
 2 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 8de693ee3..873ea7c19 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,7 +11,7 @@
       "dependencies": {
         "@aws-sdk/client-s3": "^3.435.0",
         "@itk-wasm/dicom": "^3.2.1",
-        "@kitware/vtk.js": "^28.13.0",
+        "@kitware/vtk.js": "^29.0.0",
         "@netlify/edge-functions": "^2.0.0",
         "@sentry/vue": "^7.54.0",
         "@vueuse/core": "^8.5.0",
@@ -2890,11 +2890,12 @@
       }
     },
     "node_modules/@kitware/vtk.js": {
-      "version": "28.13.0",
-      "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-28.13.0.tgz",
-      "integrity": "sha512-ZwJJ3Hp4SLssBd0yFJ04RVWutdqYfLHamBGt8uWs2j1SBAtZ75XPZ9c2PBqTGUAZKbmtRvucNpOKM8cFNQI1Eg==",
+      "version": "29.0.0",
+      "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-29.0.0.tgz",
+      "integrity": "sha512-3nRo+6fPyFT4IoRBF3Js8VlRPHPcOHf78tqQRKi+jNekeyYG2Kzx0il8hUvX9e8+hTi9rs3Dv3CduHU4e1wViA==",
       "dependencies": {
         "@babel/runtime": "7.22.11",
+        "@types/webxr": "^0.5.5",
         "commander": "9.2.0",
         "d3-scale": "4.0.2",
         "fast-deep-equal": "^3.1.3",
@@ -4457,6 +4458,26 @@
       "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.14.tgz",
       "integrity": "sha512-5d2RhCard1nQUC3aHcq/gHzWYO6K0WJmAbjO7mQJgCQKtZpgXxv1rOM6O/dBDhDYYVutk1sciOgNSe+5YyfM8A=="
     },
+    "node_modules/@types/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+      "dev": true
+    },
+    "node_modules/@types/webxr": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.7.tgz",
+      "integrity": "sha512-Rcgs5c2eNFnHp53YOjgtKfl/zWX1Y+uFGUwlSXrWcZWu3yhANRezmph4MninmqybUYT6g9ZE0aQ9QIdPkLR3Kg=="
+    },
+    "node_modules/@types/whatwg-url": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.0.tgz",
+      "integrity": "sha512-4F6szvZP3FM5HvJAmcInXBfrAhvM4tLIc8MO1nXwabG5TZVOLxVmAXRpICqXYd3lBlomSRGmLCopYV+yTocgpQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/webidl-conversions": "*"
+      }
+    },
     "node_modules/@types/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz",
@@ -24644,11 +24665,12 @@
       }
     },
     "@kitware/vtk.js": {
-      "version": "28.13.0",
-      "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-28.13.0.tgz",
-      "integrity": "sha512-ZwJJ3Hp4SLssBd0yFJ04RVWutdqYfLHamBGt8uWs2j1SBAtZ75XPZ9c2PBqTGUAZKbmtRvucNpOKM8cFNQI1Eg==",
+      "version": "29.0.0",
+      "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-29.0.0.tgz",
+      "integrity": "sha512-3nRo+6fPyFT4IoRBF3Js8VlRPHPcOHf78tqQRKi+jNekeyYG2Kzx0il8hUvX9e8+hTi9rs3Dv3CduHU4e1wViA==",
       "requires": {
         "@babel/runtime": "7.22.11",
+        "@types/webxr": "^0.5.5",
         "commander": "9.2.0",
         "d3-scale": "4.0.2",
         "fast-deep-equal": "^3.1.3",
@@ -25939,6 +25961,26 @@
       "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.14.tgz",
       "integrity": "sha512-5d2RhCard1nQUC3aHcq/gHzWYO6K0WJmAbjO7mQJgCQKtZpgXxv1rOM6O/dBDhDYYVutk1sciOgNSe+5YyfM8A=="
     },
+    "@types/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+      "dev": true
+    },
+    "@types/webxr": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.7.tgz",
+      "integrity": "sha512-Rcgs5c2eNFnHp53YOjgtKfl/zWX1Y+uFGUwlSXrWcZWu3yhANRezmph4MninmqybUYT6g9ZE0aQ9QIdPkLR3Kg=="
+    },
+    "@types/whatwg-url": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.0.tgz",
+      "integrity": "sha512-4F6szvZP3FM5HvJAmcInXBfrAhvM4tLIc8MO1nXwabG5TZVOLxVmAXRpICqXYd3lBlomSRGmLCopYV+yTocgpQ==",
+      "dev": true,
+      "requires": {
+        "@types/webidl-conversions": "*"
+      }
+    },
     "@types/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz",
diff --git a/package.json b/package.json
index 85eb107d2..b4b0c3b75 100644
--- a/package.json
+++ b/package.json
@@ -27,7 +27,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.435.0",
     "@itk-wasm/dicom": "^3.2.1",
-    "@kitware/vtk.js": "^28.13.0",
+    "@kitware/vtk.js": "^29.0.0",
     "@netlify/edge-functions": "^2.0.0",
     "@sentry/vue": "^7.54.0",
     "@vueuse/core": "^8.5.0",

From c4eec76aa8dfeb46b8ab4982bc228d0eb9ca8c39 Mon Sep 17 00:00:00 2001
From: Tom Birdsong <tom.birdsong@kitware.com>
Date: Thu, 26 Oct 2023 16:58:43 -0400
Subject: [PATCH 3/3] WIP: workaround for TypeScript `vtkViewProxy` error

---
 src/components/WebXRRendering.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/components/WebXRRendering.vue b/src/components/WebXRRendering.vue
index 5a1eecf7f..3a728968e 100644
--- a/src/components/WebXRRendering.vue
+++ b/src/components/WebXRRendering.vue
@@ -28,7 +28,7 @@ function updateXRStateFunc() {
 
 function startXRFunc(sessionType:XrSessionTypes) {
   const { viewProxy } = useViewProxy(TARGET_VIEW_ID, ViewProxyType.Volume);
-  vtkWebXRStore.xrHelper.setRenderWindow(viewProxy.value.getOpenGLRenderWindow());
+  vtkWebXRStore.xrHelper.setRenderWindow((viewProxy.value as any).getOpenGLRenderWindow());
   vtkWebXRStore.xrHelper.startXR(sessionType);
 }