Input event

js/corelink_handler.js

@@ -1,14 +1,17 @@
+"use strict";
+
 import { initXR } from "../js/immersive-pre.js"
 import { ab2str, corelink_message } from "../js/util/corelink_sender.js"
 import { initSelfAvatar } from "../js/primitive/event.js"
 import { initAvatar } from "../js/primitive/avatar.js"
 
 const workspace = 'Chalktalk'
 const protocol = 'ws'
-const datatype = ['sync', 'webrtc']
+const datatype = ['sync', 'webrtc', 'event'];
 var receiverdata = 0.0
 export var metaroomSyncSender = ""
 export var metaroomWebrtcSender = "";
+export var metaroomEventSender = "";
 export var metaroomReceiver = "";
 
 window.offlineMode = false;
@@ -85,7 +88,7 @@ const run = async () => {
             // initialize
             initSelfAvatar(metaroomSyncSender);
             // start webrtc signalling
-            window.webrtc_start();
+            // window.webrtc_start();
         }
 
         if (metaroomWebrtcSender = await corelink.createSender({
@@ -96,6 +99,14 @@ const run = async () => {
             window.webrtc_start();
         }
 
+        if (metaroomEventSender = await corelink.createSender({
+            workspace, protocol, type: 'event', echo: false, alert: true,
+        }).catch((err) => { console.log(err) })) {
+            console.log("ZH: metaroomEventSender", metaroomEventSender);
+            // start webrtc signalling
+            // window.webrtc_start();
+        }
+
         metaroomReceiver = await corelink.createReceiver({
             workspace, protocol, type: datatype, echo: true, alert: true,
         }).catch((err) => { console.log(err) })
@@ -125,12 +136,13 @@ const run = async () => {
             receiverdata = ab2str(data)
             window[streamID + '_data'] = ab2str(data)
             window.EventBus.publish(receiverdata["type"], receiverdata);
-            if (receiverdata["type"] != "avatar")
-                console.log("corelink.on('data', (streamID, data, header)", streamID, window[streamID + '_data']["type"], window[streamID + '_data'])
+            // if (receiverdata["type"] != "avatar")
+            //     console.log("corelink.on('data', (streamID, data, header)", streamID, window[streamID + '_data']["type"], window[streamID + '_data'])
 
         }).catch((err) => { console.log(err) })
 
         // Start the XR application.
+        window.initxr = true;
         initXR();
     }
 }

js/immersive-pre.js

@@ -156,6 +156,7 @@ function onRequestSession() {
 async function onSessionStarted(session) {
     session.addEventListener("end", onSessionEnded);
 
+    //Each input source should define a primary action. A primary action (which will sometimes be shortened to "select action") is a platform-specific action which responds to the user manipulating it by delivering, in order, the events selectstart, select, and selectend. Each of these events is of type XRInputSourceEvent.
     session.addEventListener("selectstart", onSelectStart);
     session.addEventListener("selectend", onSelectEnd);
     session.addEventListener("select", (ev) => {
@@ -269,6 +270,20 @@ function updateInputSources(session, frame, refSpace) {
                 headPose.transform.orientation;
             window.avatars[window.playerid].headset.matrix =
                 headPose.transform.matrix;
+
+            for (let source of session.inputSources) {
+                if (source.handedness && source.gamepad) {
+                    if (source.gamepad.buttons[3].pressed) {
+                        console.log("source.gamepad.buttons[3].pressed", source.gamepad.buttons[3].pressed);
+                    }
+                    if (source.handedness == "left")
+                        window.avatars[window.playerid].leftController.updateButtons(source.gamepad.buttons);
+                    if (source.handedness == "right")
+                        window.avatars[window.playerid].rightController.updateButtons(source.gamepad.buttons);
+                    // console.log("leftController", window.avatars[window.playerid].leftController);
+                    // console.log("rightController", window.avatars[window.playerid].rightController)
+                }
+            }
         }
     }
 }
@@ -385,6 +400,9 @@ function onXRFrame(t, frame) {
 
     updateObjects();
 
+    // ZH: save previous "source.gamepad.buttons" for two controllers,
+    // check if changes per frame
+    // send to the server if changes
     if (window.playerid) {
         const thisAvatar = window.avatars[window.playerid];
         for (let source of session.inputSources) {

js/primitive/avatar.js

@@ -2,6 +2,7 @@
 
 import { Gltf2Node } from "../render/nodes/gltf2.js";
 import { mat4, vec3 } from "../render/math/gl-matrix.js";
+import { corelink_event } from "../util/corelink_sender.js";
 
 export function initAvatar(id) {
   let headset = new Headset();
@@ -49,11 +50,13 @@ export class Avatar {
             mtx: this.leftController.matrix,
             pos: this.leftController.position,
             rot: this.leftController.orientation,
+            btn: this.leftController.toJson(),
           },
           right: {
             mtx: this.rightController.matrix,
             pos: this.rightController.position,
             rot: this.rightController.orientation,
+            btn: this.rightController.toJson(),
           },
         },
       }
@@ -68,9 +71,11 @@ export class Avatar {
       this.leftController.matrix = payload["state"]["controllers"]["left"]["mtx"];
       this.leftController.position = payload["state"]["controllers"]["left"]["pos"];
       this.leftController.orientation = payload["state"]["controllers"]["left"]["rot"];
+      this.leftController.fromJson(payload["state"]["controllers"]["left"]["btn"]);
       this.rightController.matrix = payload["state"]["controllers"]["right"]["mtx"];
       this.rightController.position = payload["state"]["controllers"]["right"]["pos"];
       this.rightController.orientation = payload["state"]["controllers"]["right"]["rot"];
+      this.rightController.fromJson(payload["state"]["controllers"]["right"]["btn"]);
       this.headset.model.visible = true;
       this.leftController.model.visible = true;
       this.rightController.model.visible = true;
@@ -107,10 +112,13 @@ export class Headset {
 export class Controller {
   constructor(handedness) {
     // this.vertices = verts;
+    this.handedness = handedness;
     this.position = vec3.fromValues(0, 0, 0);
     this.orientation = [0, 0, 0, 1];
-    // this.analog = new Button();
-    // this.trigger = new Button();
+    this.analog = new Button();
+    this.trigger = new Button();
+    this.buttons = null;
+    this.prevbuttons = null;
     // this.side = new Button();
     // this.x = new Button();
     // this.y = new Button();
@@ -124,6 +132,60 @@ export class Controller {
     this.model.scale = vec3.fromValues(1, 1, 1);
     this.model.name = "ctrl";
     this.model.visible = false;
+
+    this.updateButtons = function (newBtns) {
+      // send corelink msg if changed
+      // if (this.prevbuttons)
+      // console.log("updateButtons", this.prevbuttons[3].pressed, this.buttons[3].pressed);
+      for (let i = 0; i < 7; i++) {
+        if ((this.buttons && !this.prevbuttons && this.buttons[i].pressed)
+          || (this.prevbuttons && this.buttons[i].pressed && !this.prevbuttons[i].pressed)) {
+          corelink_event({ it: this.handedness + "trigger", op: "press" });
+        }
+        else if ((this.buttons && !this.prevbuttons && this.buttons[i].pressed)
+          || (this.prevbuttons && this.buttons[i].pressed && this.prevbuttons[i].pressed))
+          corelink_event({ it: this.handedness + "trigger", op: "drag" });
+        else if ((this.buttons && !this.prevbuttons && !this.buttons[i].pressed)
+          || (this.prevbuttons && !this.buttons[i].pressed && this.prevbuttons[i].pressed))
+          corelink_event({ it: this.handedness + "trigger", op: "release" });
+      }
+      if (this.buttons) {
+        if (!this.prevbuttons) {
+          this.prevbuttons = Array.from({ length: 7 }, (v, i) => i);
+          for (let i = 0; i < 7; i++) {
+            this.prevbuttons[i] = {
+              pressed: false,
+              touched: false,
+            };
+          }
+        }
+
+        // this.prevbuttons = Array.from(this.buttons);
+        for (let i = 0; i < 7; i++) {
+          this.prevbuttons[i].pressed = this.buttons[i].pressed;//JSON.parse(JSON.stringify(this.buttons[i]));
+          this.prevbuttons[i].touched = this.buttons[i].touched;
+        }
+      }
+
+      this.buttons = newBtns;
+    }
+
+    this.toJson = function () {
+      var jsonObj = {
+        analog: this.analog.pressed,
+        trigger: this.trigger.pressed,
+        handedness: this.handedness,
+        buttons: this.buttons,
+      }
+      // console.log("this.name", this.name, "jsonObj.name", jsonObj.name);
+      return jsonObj;
+    }
+    // fromJson will be called in event.js for unpacking the msg from the server
+    this.fromJson = function (payload) {
+      this.analog.pressed = payload["analog"];
+      this.trigger.pressed = payload["trigger"];
+      this.buttons = payload["buttons"];
+    }
   }
 }
 

js/primitive/event.js

@@ -3,7 +3,8 @@
 import { Headset, Controller, Avatar, initAvatar } from "./avatar.js";
 import { SyncObject } from "../util/object-sync.js";
 import { corelink_message } from "../util/corelink_sender.js";
-import { metaroomSyncSender, metaroomWebrtcSender } from "../corelink_handler.js"
+import { metaroomSyncSender, metaroomWebrtcSender, metaroomEventSender } from "../corelink_handler.js"
+import { left_controller_trigger, right_controller_trigger } from "../util/input_event_handler.js"
 
 export function initSelfAvatar(id) {
   if (!window.avatars) {
@@ -339,6 +340,27 @@ export function init() {
     // flags[temp] = window[temp];
   });
 
+  window.eventOwner = -1;
+  window.EventBus.subscribe("event", (json) => {
+    window.eventOwner = json["uid"];
+    // console.log("window.EventBus.subscribe('event'", json["state"], json);
+    // for (const [item, operation] of Object.entries(json["state"])) {
+    switch (json["state"]["item"]) {
+      case "lefttrigger":
+        //left trigger
+        // console.log("call left_controller_trigger", json["state"]['operation']);
+        left_controller_trigger(json["state"]['operation']);
+        break;
+      case "righttrigger":
+        //left trigger
+        right_controller_trigger(json["state"]['operation']);
+        break;
+      default:
+        break;
+    }
+    // }
+  });
+
   // on success
   // const response = {
   //   "type": "calibrate",

js/render/core/implicit_surfaces/materials.js

@@ -0,0 +1,9 @@
+export let materials = {};
+materials['black' ] = { ambient: [.0 ,.0 ,.0 ], diffuse: [0  ,0  ,0  ], specular: [.9,.9,.9,10] };
+materials['blue'  ] = { ambient: [.0 ,.0 ,.2 ], diffuse: [0  ,0  ,1  ], specular: [.9,.9,.9,10] };
+materials['brass' ] = { ambient: [.03,.02,.01], diffuse: [.03,.02,.01], specular: [.9,.6,.3,10] };
+materials['cyan'  ] = { ambient: [.0 ,.15,.15], diffuse: [0  ,.07,.07], specular: [.0,.7,.7,10] };
+materials['green' ] = { ambient: [.0 ,.2 ,.0 ], diffuse: [0  ,1  ,0  ], specular: [.9,.9,.9,10] };
+materials['red'   ] = { ambient: [.2 ,.0 ,.0 ], diffuse: [1  ,0  ,0  ], specular: [.9,.9,.9,10] };
+materials['white' ] = { ambient: [.2 ,.2 ,.2 ], diffuse: [1  ,1  ,1  ], specular: [.9,.9,.9,10] };
+materials['yellow'] = { ambient: [.2 ,.2 ,.0 ], diffuse: [1  ,1  ,0  ], specular: [.9,.9,.9,10] };

js/render/core/renderListScene.js

@@ -1,13 +1,18 @@
 "use strict";
 import { mainScene } from "../scenes/mainScene.js";
+import { corelink_event } from "../../util/corelink_sender.js";
 
-export let viewMatrix = [], time = 0, isPressed = false, isDragged = false, isReleased = true;
+export let viewMatrix = [], time = 0;
+window.isPressed = false;
+window.isDragged = false;
+window.isReleased = true;
 
 export let controllerMatrix = { left: [], right: [] };
 export let buttonState = { left: [], right: [] };
 for (let i = 0; i < 7; i++) buttonState.left[i] = buttonState.right[i] = false;
 
 const validHandedness = ["left", "right"];
+window.initxr = false;
 
 export let updateController = (avatar, buttonInfo) => {
   controllerMatrix.left = avatar.leftController.matrix;
@@ -18,34 +23,52 @@ export let updateController = (avatar, buttonInfo) => {
     const b = buttonInfo.buttons;
 
     for (let i = 0; i < 7; i++) {
-       if (b[i].pressed && !buttonState[h][i]) onPress(h, i);
-       else if (b[i].pressed && buttonState[h][i]) onDrag(h, i);
-       else if (!b[i].pressed && buttonState[h][i]) onRelease(h, i);
+      // allow local owner to react to the buttons
+      if (b[i].pressed && !buttonState[h][i]) onPress(h, i);
+      else
+        if (b[i].pressed && buttonState[h][i]) onDrag(h, i);
+        else
+          if (!b[i].pressed && buttonState[h][i]) onRelease(h, i);
 
-       // Update
-       buttonState[h][i] = b[i].pressed;
+      // Update
+      buttonState[h][i] = b[i].pressed;
     }
   }
 };
 
-let onPress = (hand, button) => {
-  console.log("pressed", hand, "button", button);
-  isPressed = true;
-  isReleased = false;
-  isDragged = false;
+export let onPress = (hand, button) => {
+
+  window.isPressed = true;
+  window.isReleased = false;
+  window.isDragged = false;
+  console.log("onPress", hand, "button", button,
+    window.isPressed, window.isReleased, window.isDragged);
+  //ZH
+  // console.log("handleSelect");
+  // corelink_event({ it: "lefttrigger", op: "press" });
 };
 
-let onDrag = (hand, button) => {
-  isDragged = true;
-  isReleased = false;
-  isPressed = false;
+export let onDrag = (hand, button) => {
+  // console.log("onDrag", hand, "button", button, isPressed, isReleased, isDragged);
+  if (window.isPressed && window.justReleased) {
+    window.isDragged = true;
+    window.isReleased = false;
+    window.isPressed = false;
+  }
 };
 
-let onRelease = (hand, button) => {
-  console.log("released", hand, "button", button);
-  isReleased = true;
-  isPressed = false;
-  isDragged = false;
+export let onRelease = (hand, button) => {
+  if (window.isDragged) {
+    window.isReleased = true;
+    window.isPressed = false;
+    window.isDragged = false;
+    console.log("onRelease", hand, "button", button,
+      window.isPressed, window.isReleased, window.isDragged);
+  }
+
+  //ZH
+  // console.log("handleSelect");
+  // corelink_event({ it: "lefttrigger", op: "release" });
 };
 
 export let getViews = (views) => {

js/render/core/scene.js

@@ -25,6 +25,7 @@ import { Node } from "./node.js";
 import { vec3, quat } from "../math/gl-matrix.js";
 import { Ray } from "../math/ray.js";
 import { getViews } from "./renderListScene.js";
+// import { corelink_event } from "../../util/corelink_sender.js";
 
 export class WebXRView extends RenderView {
   constructor(view, layer) {