ref(rrweb): allow multi touch gestures to be shown for mobile replays

packages/rrweb/src/replay/index.ts

@@ -63,6 +63,8 @@
   styleSheetRuleData,
   styleDeclarationData,
   adoptedStyleSheetData,
+  mouseInteractionData,
+  mousemoveData,
 } from '@sentry-internal/rrweb-types';
 import {
   polyfill,
@@ -98,7 +100,14 @@
   strokeStyle: 'red',
 } as const;
 
-function indicatesTouchDevice(e: eventWithTime) {
+type incrementalSnapshotEventWithTime = incrementalSnapshotEvent & {
+  timestamp: number;
+  delay?: number;
+};
+
+function indicatesTouchDevice(
+  e: eventWithTime,
+): e is incrementalSnapshotEventWithTime {
   return (
     e.type == EventType.IncrementalSnapshot &&
     (e.data.source == IncrementalSource.TouchMove ||
@@ -107,6 +116,14 @@
   );
 }
 
+function getPointerId(
+  d: incrementalData | mousemoveData | mouseInteractionData,
+): number {
+  const pointerId =
+    'pointerId' in d && typeof d.pointerId === 'number' ? d.pointerId : -1;
+  return pointerId;
+}
+
 export class Replayer {
   public wrapper: HTMLDivElement;
   public iframe: HTMLIFrameElement;
@@ -123,9 +140,7 @@
   public usingVirtualDom = false;
   public virtualDom: RRDocument = new RRDocument();
 
-  private mouse: HTMLDivElement;
   private mouseTail: HTMLCanvasElement | null = null;
-  private tailPositions: Array<{ x: number; y: number }> = [];
 
   private emitter: Emitter = mitt();
 
@@ -148,8 +163,16 @@
 
   private newDocumentQueue: addedNodeMutation[] = [];
 
-  private mousePos: mouseMovePos | null = null;
-  private touchActive: boolean | null = null;
+  // Map of pointer ID to the unique vars used to show pointers and their movements
+  private pointers: Record<
+    number,
+    {
+      touchActive: boolean | null;
+      pointerEl: HTMLDivElement;
+      tailPositions: Array<{ x: number; y: number }>;
+      pointerPosition: mouseMovePos | null;
+    }
+  > = {};
   private lastMouseDownEvent: [Node, Event] | null = null;
 
   // Keep the rootNode of the last hovered element. So  when hovering a new element, we can remove the last hovered element's :hover style.
@@ -293,23 +316,32 @@
         this.adoptedStyleSheets = [];
       }
 
-      if (this.mousePos) {
-        this.moveAndHover(
-          this.mousePos.x,
-          this.mousePos.y,
-          this.mousePos.id,
-          true,
-          this.mousePos.debugData,
-        );
-        this.mousePos = null;
-      }
+      for (const [
+        pointerId,
+        { pointerPosition, touchActive },
+      ] of Object.entries(this.pointers)) {
+        const id = parseInt(pointerId);
+        const pointer = this.pointers[id];
+
+        if (pointerPosition) {
+          this.moveAndHover(
+            pointerPosition.x,
+            pointerPosition.y,
+            pointerPosition.id,
+            true,
+            pointerPosition.debugData,
+            id,
+          );
+          pointer.pointerPosition = null;
+        }
 
-      if (this.touchActive === true) {
-        this.mouse.classList.add('touch-active');
-      } else if (this.touchActive === false) {
-        this.mouse.classList.remove('touch-active');
+        if (touchActive === true) {
+          pointer.pointerEl.classList.add('touch-active');
+        } else if (touchActive === false) {
+          pointer.pointerEl.classList.remove('touch-active');
+        }
+        pointer.touchActive = null;
       }
-      this.touchActive = null;
 
       if (this.lastMouseDownEvent) {
         const [target, event] = this.lastMouseDownEvent;
@@ -402,9 +434,21 @@
         );
       }, 1);
     }
-    if (this.service.state.context.events.find(indicatesTouchDevice)) {
-      this.mouse.classList.add('touch-device');
+  }
+
+  private createPointer(pointerId: number, event: eventWithTime) {
+    const newMouse = document.createElement('div');
+    newMouse.classList.add('replayer-mouse');
+    this.pointers[pointerId] = {
+      touchActive: null,
+      pointerEl: newMouse,
+      tailPositions: [],
+      pointerPosition: null,
+    };
+    if (indicatesTouchDevice(event)) {
+      newMouse.classList.add('touch-device');
     }
+    this.wrapper.appendChild(newMouse);
   }
 
   public on(event: string, handler: Handler) {
@@ -541,9 +585,7 @@
     const event = this.config.unpackFn
       ? this.config.unpackFn(rawEvent as string)
       : (rawEvent as eventWithTime);
-    if (indicatesTouchDevice(event)) {
-      this.mouse.classList.add('touch-device');
-    }
+
     void Promise.resolve().then(() =>
       this.service.send({ type: 'ADD_EVENT', payload: { event } }),
     );
@@ -572,10 +614,6 @@
     this.wrapper.classList.add('replayer-wrapper');
     this.config.root.appendChild(this.wrapper);
 
-    this.mouse = document.createElement('div');
-    this.mouse.classList.add('replayer-mouse');
-    this.wrapper.appendChild(this.mouse);
-
     if (this.config.mouseTail !== false) {
       this.mouseTail = document.createElement('canvas');
       this.mouseTail.classList.add('replayer-mouse-tail');
@@ -887,7 +925,7 @@
         sn?.type === NodeType.Element &&
         sn?.tagName.toUpperCase() === 'HTML'
       ) {
         const { documentElement, head } = iframeEl.contentDocument!;
         this.insertStyleRules(
           documentElement as HTMLElement | RRElement,
           head as HTMLElement | RRElement,
@@ -906,14 +944,14 @@
     };
 
     buildNodeWithSN(mutation.node, {
       doc: iframeEl.contentDocument! as Document,
       mirror: mirror as Mirror,
       hackCss: true,
       skipChild: false,
       afterAppend,
       cache: this.cache,
     });
     afterAppend(iframeEl.contentDocument! as Document, mutation.node.id);
 
     for (const { mutationInQueue, builtNode } of collected) {
       this.attachDocumentToIframe(mutationInQueue, builtNode);
@@ -1000,7 +1038,7 @@
    * pause when there are some canvas drawImage args need to be loaded
    */
   private async preloadAllImages(): Promise<void[]> {
     let beforeLoadState = this.service.state;
     const stateHandler = () => {
       beforeLoadState = this.service.state;
     };
@@ -1035,8 +1073,8 @@
       const ctx = canvas.getContext('2d');
       const imgd = ctx?.createImageData(canvas.width, canvas.height);
       let d = imgd?.data;
       d = JSON.parse(data.args[0]) as Uint8ClampedArray;
       ctx?.putImageData(imgd!, 0, 0);
     }
   }
   private async deserializeAndPreloadCanvasEvents(
@@ -1085,10 +1123,17 @@
       }
       case IncrementalSource.Drag:
       case IncrementalSource.TouchMove:
-      case IncrementalSource.MouseMove:
+      case IncrementalSource.MouseMove: {
+        const pointerId = getPointerId(d);
+        if (!this.pointers[pointerId]) {
+          this.createPointer(pointerId, e);
+        }
+
+        const pointer = this.pointers[pointerId];
+
         if (isSync) {
           const lastPosition = d.positions[d.positions.length - 1];
-          this.mousePos = {
+          pointer.pointerPosition = {
             x: lastPosition.x,
             y: lastPosition.y,
             id: lastPosition.id,
@@ -1098,7 +1143,7 @@
           d.positions.forEach((p) => {
             const action = {
               doAction: () => {
-                this.moveAndHover(p.x, p.y, p.id, isSync, d);
+                this.moveAndHover(p.x, p.y, p.id, isSync, d, pointerId);
               },
               delay:
                 p.timeOffset +
@@ -1117,7 +1162,15 @@
           });
         }
         break;
+      }
       case IncrementalSource.MouseInteraction: {
+        const pointerId = getPointerId(d);
+        if (!this.pointers[pointerId]) {
+          this.createPointer(pointerId, e);
+        }
+
+        const pointer = this.pointers[pointerId];
+
         /**
          * Same as the situation of missing input target.
          */
@@ -1154,16 +1207,18 @@
           case MouseInteractions.MouseUp:
             if (isSync) {
               if (d.type === MouseInteractions.TouchStart) {
-                this.touchActive = true;
+                pointer.touchActive = true;
               } else if (d.type === MouseInteractions.TouchEnd) {
-                this.touchActive = false;
+                pointer.touchActive = false;
+                pointer.pointerEl.remove();
+                delete this.pointers[pointerId];
               }
               if (d.type === MouseInteractions.MouseDown) {
                 this.lastMouseDownEvent = [target, event];
               } else if (d.type === MouseInteractions.MouseUp) {
                 this.lastMouseDownEvent = null;
               }
-              this.mousePos = {
+              pointer.pointerPosition = {
                 x: d.x || 0,
                 y: d.y || 0,
                 id: d.id,
@@ -1172,9 +1227,9 @@
             } else {
               if (d.type === MouseInteractions.TouchStart) {
                 // don't draw a trail as user has lifted finger and is placing at a new point
-                this.tailPositions.length = 0;
+                pointer.tailPositions.length = 0;
               }
-              this.moveAndHover(d.x || 0, d.y || 0, d.id, isSync, d);
+              this.moveAndHover(d.x || 0, d.y || 0, d.id, isSync, d, pointerId);
               if (d.type === MouseInteractions.Click) {
                 /*
                  * don't want target.click() here as could trigger an iframe navigation
@@ -1184,14 +1239,15 @@
                  * removal and addition of .active class (along with void line to trigger repaint)
                  * triggers the 'click' css animation in styles/style.css
                  */
-                this.mouse.classList.remove('active');
-                void this.mouse.offsetWidth;
-                this.mouse.classList.add('active');
+                pointer.pointerEl.classList.remove('active');
+                void pointer.pointerEl.offsetWidth;
+                pointer.pointerEl.classList.add('active');
               } else if (d.type === MouseInteractions.TouchStart) {
-                void this.mouse.offsetWidth; // needed for the position update of moveAndHover to apply without the .touch-active transition
-                this.mouse.classList.add('touch-active');
+                void pointer.pointerEl.offsetWidth; // needed for the position update of moveAndHover to apply without the .touch-active transition
+                pointer.pointerEl.classList.add('touch-active');
               } else if (d.type === MouseInteractions.TouchEnd) {
-                this.mouse.classList.remove('touch-active');
+                pointer.pointerEl.remove();
+                delete this.pointers[pointerId];
               } else {
                 // for MouseDown & MouseUp also invoke default behavior
                 target.dispatchEvent(event);
@@ -1200,9 +1256,9 @@
             break;
           case MouseInteractions.TouchCancel:
             if (isSync) {
-              this.touchActive = false;
+              pointer.touchActive = false;
             } else {
-              this.mouse.classList.remove('touch-active');
+              pointer.pointerEl.classList.remove('touch-active');
             }
             break;
           default:
@@ -1373,7 +1429,7 @@
     // Only apply virtual dom optimization if the fast-forward process has node mutation. Because the cost of creating a virtual dom tree and executing the diff algorithm is usually higher than directly applying other kind of events.
     if (this.config.useVirtualDom && !this.usingVirtualDom && isSync) {
       this.usingVirtualDom = true;
       buildFromDom(this.iframe.contentDocument!, this.mirror, this.virtualDom);
       // If these legacy missing nodes haven't been resolved, they should be converted to virtual nodes.
       if (Object.keys(this.legacy_missingNodeRetryMap).length) {
         for (const key in this.legacy_missingNodeRetryMap) {
@@ -1490,7 +1546,7 @@
         // If the parent is attached a shadow dom after it's created, it won't have a shadow root.
         if (!hasShadowRoot(parent)) {
           (parent as Element | RRElement).attachShadow({ mode: 'open' });
           parent = (parent as Element | RRElement).shadowRoot! as Node | RRNode;
         } else parent = parent.shadowRoot as Node | RRNode;
       }
 
@@ -2080,6 +2136,7 @@
     id: number,
     isSync: boolean,
     debugData: incrementalData,
+    pointerId: number,
   ) {
     const target = this.mirror.getNode(id);
     if (!target) {
@@ -2090,15 +2147,16 @@
     const _x = x * base.absoluteScale + base.x;
     const _y = y * base.absoluteScale + base.y;
 
-    this.mouse.style.left = `${_x}px`;
-    this.mouse.style.top = `${_y}px`;
+    const pointer = this.pointers[pointerId];
+    pointer.pointerEl.style.left = `${_x}px`;
+    pointer.pointerEl.style.top = `${_y}px`;
     if (!isSync) {
-      this.drawMouseTail({ x: _x, y: _y });
+      this.drawMouseTail({ x: _x, y: _y }, pointerId);
     }
     this.hoverElements(target as Element);
   }
 
-  private drawMouseTail(position: { x: number; y: number }) {
+  private drawMouseTail(position: { x: number; y: number }, pointerId: number) {
     if (!this.mouseTail) {
       return;
     }
@@ -2108,29 +2166,35 @@
         ? defaultMouseTailConfig
         : Object.assign({}, defaultMouseTailConfig, this.config.mouseTail);
 
+    const pointer = this.pointers[pointerId];
+
     const draw = () => {
       if (!this.mouseTail) {
         return;
       }
       const ctx = this.mouseTail.getContext('2d');
-      if (!ctx || !this.tailPositions.length) {
+      if (!ctx || !pointer.tailPositions.length) {
         return;
       }
       ctx.clearRect(0, 0, this.mouseTail.width, this.mouseTail.height);
       ctx.beginPath();
       ctx.lineWidth = lineWidth;
       ctx.lineCap = lineCap;
       ctx.strokeStyle = strokeStyle;
-      ctx.moveTo(this.tailPositions[0].x, this.tailPositions[0].y);
-      this.tailPositions.forEach((p) => ctx.lineTo(p.x, p.y));
+      ctx.moveTo(pointer.tailPositions[0].x, pointer.tailPositions[0].y);
+      pointer.tailPositions.forEach((p) => ctx.lineTo(p.x, p.y));
       ctx.stroke();
     };
 
-    this.tailPositions.push(position);
+    pointer.tailPositions.push(position);
     draw();
     setTimeout(() => {
-      this.tailPositions = this.tailPositions.filter((p) => p !== position);
-      draw();
+      if (pointerId in this.pointers) {
+        pointer.tailPositions = pointer.tailPositions.filter(
+          (p) => p !== position,
+        );
+        draw();
+      }
     }, duration / this.speedService.state.context.timer.speed);
   }
 

packages/rrweb/test/__snapshots__/replayer.test.ts.snap

@@ -8,7 +8,6 @@ exports[`replayer can fast forward past StyleSheetRule changes on virtual elemen
   </head>
   <body>
     <div class=\\"replayer-wrapper\\">
-      <div class=\\"replayer-mouse\\"></div>
       <canvas
         class=\\"replayer-mouse-tail\\"
         width=\\"1000\\"
@@ -88,7 +87,6 @@ exports[`replayer can fast forward past StyleSheetRule deletion on virtual eleme
   </head>
   <body>
     <div class=\\"replayer-wrapper\\">
-      <div class=\\"replayer-mouse\\"></div>
       <canvas
         class=\\"replayer-mouse-tail\\"
         width=\\"1000\\"
@@ -166,7 +164,6 @@ exports[`replayer can handle removing style elements 1`] = `
   </head>
   <body>
     <div class=\\"replayer-wrapper\\">
-      <div class=\\"replayer-mouse\\"></div>
       <canvas
         class=\\"replayer-mouse-tail\\"
         width=\\"1000\\"
@@ -214,7 +211,6 @@ exports[`replayer replays same timestamp events in correct order (with addAction
   </head>
   <body>
     <div class=\\"replayer-wrapper\\">
-      <div class=\\"replayer-mouse\\"></div>
       <canvas
         class=\\"replayer-mouse-tail\\"
         width=\\"1000\\"
@@ -265,7 +261,6 @@ exports[`replayer replays same timestamp events in correct order 1`] = `
   </head>
   <body>
     <div class=\\"replayer-wrapper\\">
-      <div class=\\"replayer-mouse\\"></div>
       <canvas
         class=\\"replayer-mouse-tail\\"
         width=\\"1000\\"