whotracksme · philipp-classen · Oct 10, 2023 · Oct 3, 2023 · Oct 3, 2023 · Oct 9, 2023
diff --git a/packages/reporting/example/index.js b/packages/reporting/example/index.js
@@ -16,6 +16,13 @@ import {
 } from '../src/index.js';
 import rules from './rules.json';
 
+(chrome.action || chrome.browserAction).onClicked.addListener(() => {
+  chrome.tabs.create({
+    active: true,
+    url: chrome.runtime.getURL('inspector/index.html'),
+  });
+});
+
 setLogLevel('debug');
 
 const storage = {
@@ -71,14 +78,18 @@ const requestReporter = new RequestReporter(config.request, {
   getBrowserInfo: () => ({ name: 'xx' }),
 });
 
-chrome.runtime.onMessage.addListener((request, sender) => {
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
   if (request.action === 'mousedown') {
     requestReporter.recordClick(
       request.event,
       request.context,
       request.href,
       sender,
     );
+  } else if (request.action === 'debug') {
+    sendResponse({
+      tabs: [...webRequestPipeline.pageStore.tabs._inMemoryMap.values()],
+    });
   }
 });
 
@@ -91,5 +102,6 @@ chrome.runtime.onMessage.addListener((request, sender) => {
   await requestReporter.init();
 })();
 
+globalThis.webRequestPipeline = webRequestPipeline;
 globalThis.urlReporter = urlReporter;
 globalThis.requestReporter = requestReporter;
diff --git a/packages/reporting/example/inspector/index.html b/packages/reporting/example/inspector/index.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>wtm/reporting inspector</title>
+  <script type="module" src="./index.js"></script>
+</head>
+<body>
+  <h1>WTM Reporting Inspector</h1>
+  <label for="filter">Filter:</label><input id="filter" type="text"/>
+  <section>
+    <h2>Page Store</h2>
+    <pre id="page-store"></pre>
+  </section>
+
+</body>
+</html>
diff --git a/packages/reporting/example/inspector/index.js b/packages/reporting/example/inspector/index.js
@@ -0,0 +1,27 @@
+const browser = globalThis.browser || globalThis.chrome;
+const $container = document.querySelector('#page-store');
+const $filter = document.querySelector('#filter');
+
+$filter.value = localStorage.filter || '';
+$filter.addEventListener('input', () => {
+  localStorage.filter = $filter.value;
+});
+
+async function render() {
+  let { tabs } = await browser.runtime.sendMessage({ action: 'debug' });
+
+  if (localStorage.filter) {
+    const matchById = tabs.find(
+      (tab) => tab.id === Number(localStorage.filter),
+    );
+
+    tabs = matchById
+      ? [matchById]
+      : tabs.filter((tab) => tab.url.includes(localStorage.filter));
+  }
+  $container.innerHTML = JSON.stringify(tabs, null, 2);
+}
+
+await render();
+
+setInterval(render, 500);
diff --git a/packages/reporting/example/manifests/chromium.json b/packages/reporting/example/manifests/chromium.json
@@ -13,6 +13,9 @@
     "storage",
     "windows"
   ],
+  "action": {
+    "default_title": "WTM/Reporting Inspector"
+  },
   "background": {
     "service_worker": "index.bundle.js",
     "type": "module"

diff --git a/packages/reporting/example/manifests/firefox.json b/packages/reporting/example/manifests/firefox.json
@@ -16,6 +16,10 @@
     "ws://*/*",
     "wss://*/*"
   ],
+  "browser_action": {
+    "default_title": "WTM/Reporting Inspector",
+    "default_area": "navbar"
+  },
   "content_scripts": [
     {
       "matches": ["http://*/*", "https://*/*"],

diff --git a/packages/reporting/example/tests/test1.html b/packages/reporting/example/tests/test1.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>wtm/reporting test1</title>
+</head>
+<body>
+  <a href="https://ghostery.com" id="a">go to ghostery.com (forces process switch on Firefox)</a>
+</body>
+</html>
diff --git a/packages/reporting/src/request/page-telemetry.js b/packages/reporting/src/request/page-telemetry.js
@@ -38,7 +38,5 @@ export default function buildPageLoadObject(page) {
     tsv: page.tsv,
     tsv_id: page.tsvId !== undefined,
     frames: {},
-    cmp: page.annotations.cmp,
-    hiddenElements: page.annotations.hiddenElements,
   };
 }
diff --git a/packages/reporting/src/webrequest-pipeline/page-store.js b/packages/reporting/src/webrequest-pipeline/page-store.js
@@ -37,9 +37,6 @@ export default class PageStore {
     chrome.tabs.onActivated.addListener(this.onTabActivated);
     chrome.webNavigation.onBeforeNavigate.addListener(this.onBeforeNavigate);
     chrome.webNavigation.onCommitted.addListener(this.onNavigationCommitted);
-    chrome.webNavigation.onDOMContentLoaded.addListener(
-      this.onNavigationLoaded,
-    );
     chrome.webNavigation.onCompleted.addListener(this.onNavigationComplete);
     if (chrome.windows && chrome.windows.onFocusChanged) {
       chrome.windows.onFocusChanged.addListener(this.onWindowFocusChanged);
@@ -62,9 +59,6 @@ export default class PageStore {
     chrome.tabs.onActivated.removeListener(this.onTabActivated);
     chrome.webNavigation.onBeforeNavigate.removeListener(this.onBeforeNavigate);
     chrome.webNavigation.onCommitted.removeListener(this.onNavigationCommitted);
-    chrome.webNavigation.onDOMContentLoaded.removeListener(
-      this.onNavigationLoaded,
-    );
     chrome.webNavigation.onCompleted.removeListener(this.onNavigationComplete);
     if (chrome.windows && chrome.windows.onFocusChanged) {
       chrome.windows.onFocusChanged.removeListener(this.onWindowFocusChanged);
@@ -147,9 +141,18 @@ export default class PageStore {
   };
 
   onBeforeNavigate = (details) => {
-    const { frameId, tabId, url } = details;
+    const { frameId, tabId, url, timeStamp } = details;
     const tabContext = this.tabs.get(tabId);
     if (frameId === 0) {
+      // ignore duplicated onBeforeNavigate https://bugzilla.mozilla.org/show_bug.cgi?id=1732564
+      if (
+        tabContext &&
+        tabContext.id === tabId &&
+        tabContext.url === url &&
+        tabContext.created + 200 > timeStamp
+      ) {
+        return;
+      }
       // We are starting a navigation to a new page - if the previous page is complete (i.e. fully
       // loaded), stage it before we create the new page info.
       if (tabContext && tabContext.state === PAGE_LOADING_STATE.COMPLETE) {
@@ -162,6 +165,7 @@ export default class PageStore {
         active: false,
         url,
         incognito: tabContext ? tabContext.isPrivate : false,
+        created: timeStamp,
       });
       nextContext.previous = tabContext;
       this.tabs.set(tabId, nextContext);
@@ -180,13 +184,6 @@ export default class PageStore {
     }
   };
 
-  onNavigationLoaded = ({ frameId, tabId }) => {
-    const tabContext = this.tabs.get(tabId);
-    if (frameId === 0 && tabContext) {
-      tabContext.updateState(PAGE_LOADING_STATE.LOADED);
-    }
-  };
-
   onNavigationComplete = ({ frameId, tabId }) => {
     const tabContext = this.tabs.get(tabId);
     if (frameId === 0 && tabContext) {
@@ -244,7 +241,8 @@ export default class PageStore {
     };
   };
 
-  getPageForRequest({ tabId, frameId, originUrl, type, initiator }) {
+  getPageForRequest(context) {
+    const { tabId, frameId, originUrl, type, initiator } = context;
     const tab = this.tabs.get(tabId);
     if (!tab) {
       return null;
@@ -260,6 +258,7 @@ export default class PageStore {
 
     const couldBePreviousPage =
       frameId === 0 && type !== 'main_frame' && tab.previous;
+
     // for main frame requests: check if the origin url is from the previous page (Firefox)
     if (
       couldBePreviousPage &&

diff --git a/packages/reporting/src/webrequest-pipeline/page.js b/packages/reporting/src/webrequest-pipeline/page.js
@@ -13,18 +13,17 @@ export const PAGE_LOADING_STATE = {
   CREATED: 'created',
   NAVIGATING: 'navigating',
   COMMITTED: 'committed',
-  LOADED: 'loaded',
   COMPLETE: 'complete',
 };
 
 export default class Page {
-  constructor({ id, active, url, incognito }) {
+  constructor({ id, active, url, incognito, created }) {
     this.id = id || 0;
     this.url = url;
     this.isRedirect = false;
     this.isPrivate = incognito;
     this.isPrivateServer = false;
-    this.created = Date.now();
+    this.created = created || Date.now();
     this.destroyed = null;
     this.lastRequestId = null;
     this.frames = {