From d5fd91b1bd72ee08351f395e4975ec72363ec393 Mon Sep 17 00:00:00 2001 From: Krzysztof Modras Date: Tue, 3 Oct 2023 18:15:58 +0200 Subject: [PATCH 1/4] Testing setup --- packages/reporting/example/index.js | 14 +++++++++- .../reporting/example/inspector/index.html | 17 ++++++++++++ packages/reporting/example/inspector/index.js | 27 +++++++++++++++++++ .../reporting/example/manifests/chromium.json | 3 +++ .../reporting/example/manifests/firefox.json | 4 +++ packages/reporting/example/tests/test1.html | 10 +++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 packages/reporting/example/inspector/index.html create mode 100644 packages/reporting/example/inspector/index.js create mode 100644 packages/reporting/example/tests/test1.html diff --git a/packages/reporting/example/index.js b/packages/reporting/example/index.js index 534c9012..1b6b71e4 100644 --- a/packages/reporting/example/index.js +++ b/packages/reporting/example/index.js @@ -16,6 +16,13 @@ import { } from '../src/index.js'; import rules from './rules.json'; +(chrome.action || chrome.browserAction).onClicked.addListener(() => { + chrome.tabs.create({ + active: true, + url: chrome.runtime.getURL('inspector/index.html'), + }); +}); + setLogLevel('debug'); const storage = { @@ -71,7 +78,7 @@ const requestReporter = new RequestReporter(config.request, { getBrowserInfo: () => ({ name: 'xx' }), }); -chrome.runtime.onMessage.addListener((request, sender) => { +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { if (request.action === 'mousedown') { requestReporter.recordClick( request.event, @@ -79,6 +86,10 @@ chrome.runtime.onMessage.addListener((request, sender) => { request.href, sender, ); + } else if (request.action === 'debug') { + sendResponse({ + tabs: [...webRequestPipeline.pageStore.tabs._inMemoryMap.values()], + }); } }); @@ -91,5 +102,6 @@ chrome.runtime.onMessage.addListener((request, sender) => { await requestReporter.init(); })(); +globalThis.webRequestPipeline = webRequestPipeline; globalThis.urlReporter = urlReporter; globalThis.requestReporter = requestReporter; diff --git a/packages/reporting/example/inspector/index.html b/packages/reporting/example/inspector/index.html new file mode 100644 index 00000000..ce05a01d --- /dev/null +++ b/packages/reporting/example/inspector/index.html @@ -0,0 +1,17 @@ + + + + + wtm/reporting inspector + + + +

WTM Reporting Inspector

+ +
+

Page Store

+ 

+  

+
+
+
diff --git a/packages/reporting/example/inspector/index.js b/packages/reporting/example/inspector/index.js
new file mode 100644
index 00000000..0b27f986
--- /dev/null
+++ b/packages/reporting/example/inspector/index.js
@@ -0,0 +1,27 @@
+const browser = globalThis.browser || globalThis.chrome;
+const $container = document.querySelector('#page-store');
+const $filter = document.querySelector('#filter');
+
+$filter.value = localStorage.filter || '';
+$filter.addEventListener('input', () => {
+  localStorage.filter = $filter.value;
+});
+
+async function render() {
+  let { tabs } = await browser.runtime.sendMessage({ action: 'debug' });
+
+  if (localStorage.filter) {
+    const matchById = tabs.find(
+      (tab) => tab.id === Number(localStorage.filter),
+    );
+
+    tabs = matchById
+      ? [matchById]
+      : tabs.filter((tab) => tab.url.includes(localStorage.filter));
+  }
+  $container.innerHTML = JSON.stringify(tabs, null, 2);
+}
+
+await render();
+
+setInterval(render, 500);
diff --git a/packages/reporting/example/manifests/chromium.json b/packages/reporting/example/manifests/chromium.json
index 6c0a075b..25b45e78 100644
--- a/packages/reporting/example/manifests/chromium.json
+++ b/packages/reporting/example/manifests/chromium.json
@@ -13,6 +13,9 @@
     "storage",
     "windows"
   ],
+  "action": {
+    "default_title": "WTM/Reporting Inspector"
+  },
   "background": {
     "service_worker": "index.bundle.js",
     "type": "module"
diff --git a/packages/reporting/example/manifests/firefox.json b/packages/reporting/example/manifests/firefox.json
index ade42203..2124d051 100644
--- a/packages/reporting/example/manifests/firefox.json
+++ b/packages/reporting/example/manifests/firefox.json
@@ -16,6 +16,10 @@
     "ws://*/*",
     "wss://*/*"
   ],
+  "browser_action": {
+    "default_title": "WTM/Reporting Inspector",
+    "default_area": "navbar"
+  },
   "content_scripts": [
     {
       "matches": ["http://*/*", "https://*/*"],
diff --git a/packages/reporting/example/tests/test1.html b/packages/reporting/example/tests/test1.html
new file mode 100644
index 00000000..97723912
--- /dev/null
+++ b/packages/reporting/example/tests/test1.html
@@ -0,0 +1,10 @@
+
+
+
+  
+  wtm/reporting test1
+
+
+  go to ghostery.com (forces process switch on Firefox)
+
+

From 3e9b3d205582e861ff66d85e9f43b468f866b0d4 Mon Sep 17 00:00:00 2001
From: Krzysztof Modras 
Date: Tue, 3 Oct 2023 18:31:10 +0200
Subject: [PATCH 2/4] Remove dead code

---
 packages/reporting/src/request/page-telemetry.js    |  2 --
 .../reporting/src/webrequest-pipeline/page-store.js | 13 -------------
 packages/reporting/src/webrequest-pipeline/page.js  |  1 -
 3 files changed, 16 deletions(-)

diff --git a/packages/reporting/src/request/page-telemetry.js b/packages/reporting/src/request/page-telemetry.js
index fecc4ad9..ed764737 100644
--- a/packages/reporting/src/request/page-telemetry.js
+++ b/packages/reporting/src/request/page-telemetry.js
@@ -38,7 +38,5 @@ export default function buildPageLoadObject(page) {
     tsv: page.tsv,
     tsv_id: page.tsvId !== undefined,
     frames: {},
-    cmp: page.annotations.cmp,
-    hiddenElements: page.annotations.hiddenElements,
   };
 }
diff --git a/packages/reporting/src/webrequest-pipeline/page-store.js b/packages/reporting/src/webrequest-pipeline/page-store.js
index 44edc73b..793e932a 100644
--- a/packages/reporting/src/webrequest-pipeline/page-store.js
+++ b/packages/reporting/src/webrequest-pipeline/page-store.js
@@ -37,9 +37,6 @@ export default class PageStore {
     chrome.tabs.onActivated.addListener(this.onTabActivated);
     chrome.webNavigation.onBeforeNavigate.addListener(this.onBeforeNavigate);
     chrome.webNavigation.onCommitted.addListener(this.onNavigationCommitted);
-    chrome.webNavigation.onDOMContentLoaded.addListener(
-      this.onNavigationLoaded,
-    );
     chrome.webNavigation.onCompleted.addListener(this.onNavigationComplete);
     if (chrome.windows && chrome.windows.onFocusChanged) {
       chrome.windows.onFocusChanged.addListener(this.onWindowFocusChanged);
@@ -62,9 +59,6 @@ export default class PageStore {
     chrome.tabs.onActivated.removeListener(this.onTabActivated);
     chrome.webNavigation.onBeforeNavigate.removeListener(this.onBeforeNavigate);
     chrome.webNavigation.onCommitted.removeListener(this.onNavigationCommitted);
-    chrome.webNavigation.onDOMContentLoaded.removeListener(
-      this.onNavigationLoaded,
-    );
     chrome.webNavigation.onCompleted.removeListener(this.onNavigationComplete);
     if (chrome.windows && chrome.windows.onFocusChanged) {
       chrome.windows.onFocusChanged.removeListener(this.onWindowFocusChanged);
@@ -180,13 +174,6 @@ export default class PageStore {
     }
   };
 
-  onNavigationLoaded = ({ frameId, tabId }) => {
-    const tabContext = this.tabs.get(tabId);
-    if (frameId === 0 && tabContext) {
-      tabContext.updateState(PAGE_LOADING_STATE.LOADED);
-    }
-  };
-
   onNavigationComplete = ({ frameId, tabId }) => {
     const tabContext = this.tabs.get(tabId);
     if (frameId === 0 && tabContext) {
diff --git a/packages/reporting/src/webrequest-pipeline/page.js b/packages/reporting/src/webrequest-pipeline/page.js
index 7413b239..63575ca5 100644
--- a/packages/reporting/src/webrequest-pipeline/page.js
+++ b/packages/reporting/src/webrequest-pipeline/page.js
@@ -13,7 +13,6 @@ export const PAGE_LOADING_STATE = {
   CREATED: 'created',
   NAVIGATING: 'navigating',
   COMMITTED: 'committed',
-  LOADED: 'loaded',
   COMPLETE: 'complete',
 };
 

From d7a6125b40fde01c13ed27a816a798cc626ec8c1 Mon Sep 17 00:00:00 2001
From: Krzysztof Modras 
Date: Mon, 9 Oct 2023 17:33:29 +0200
Subject: [PATCH 3/4] Page store: prevent Firefox bug breaking previous page
 chain

---
 .../src/webrequest-pipeline/page-store.js        | 16 ++++++++++++++--
 .../reporting/src/webrequest-pipeline/page.js    |  4 ++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/packages/reporting/src/webrequest-pipeline/page-store.js b/packages/reporting/src/webrequest-pipeline/page-store.js
index 793e932a..119cbcf2 100644
--- a/packages/reporting/src/webrequest-pipeline/page-store.js
+++ b/packages/reporting/src/webrequest-pipeline/page-store.js
@@ -141,9 +141,18 @@ export default class PageStore {
   };
 
   onBeforeNavigate = (details) => {
-    const { frameId, tabId, url } = details;
+    const { frameId, tabId, url, timeStamp } = details;
     const tabContext = this.tabs.get(tabId);
     if (frameId === 0) {
+      // ignore dupilcated onBeforeNavigate https://bugzilla.mozilla.org/show_bug.cgi?id=1732564
+      if (
+        tabContext &&
+        tabContext.id === tabId &&
+        tabContext.url === url &&
+        tabContext.created + 200 > timeStamp
+      ) {
+        return;
+      }
       // We are starting a navigation to a new page - if the previous page is complete (i.e. fully
       // loaded), stage it before we create the new page info.
       if (tabContext && tabContext.state === PAGE_LOADING_STATE.COMPLETE) {
@@ -156,6 +165,7 @@ export default class PageStore {
         active: false,
         url,
         incognito: tabContext ? tabContext.isPrivate : false,
+        created: timeStamp,
       });
       nextContext.previous = tabContext;
       this.tabs.set(tabId, nextContext);
@@ -231,7 +241,8 @@ export default class PageStore {
     };
   };
 
-  getPageForRequest({ tabId, frameId, originUrl, type, initiator }) {
+  getPageForRequest(context) {
+    const { tabId, frameId, originUrl, type, initiator } = context;
     const tab = this.tabs.get(tabId);
     if (!tab) {
       return null;
@@ -247,6 +258,7 @@ export default class PageStore {
 
     const couldBePreviousPage =
       frameId === 0 && type !== 'main_frame' && tab.previous;
+
     // for main frame requests: check if the origin url is from the previous page (Firefox)
     if (
       couldBePreviousPage &&
diff --git a/packages/reporting/src/webrequest-pipeline/page.js b/packages/reporting/src/webrequest-pipeline/page.js
index 63575ca5..75e657d4 100644
--- a/packages/reporting/src/webrequest-pipeline/page.js
+++ b/packages/reporting/src/webrequest-pipeline/page.js
@@ -17,13 +17,13 @@ export const PAGE_LOADING_STATE = {
 };
 
 export default class Page {
-  constructor({ id, active, url, incognito }) {
+  constructor({ id, active, url, incognito, created }) {
     this.id = id || 0;
     this.url = url;
     this.isRedirect = false;
     this.isPrivate = incognito;
     this.isPrivateServer = false;
-    this.created = Date.now();
+    this.created = created || Date.now();
     this.destroyed = null;
     this.lastRequestId = null;
     this.frames = {

From 17940491966cb2821c094c8098fd06b7dcdbaa5a Mon Sep 17 00:00:00 2001
From: Krzysztof Modras 
Date: Tue, 10 Oct 2023 15:48:52 +0200
Subject: [PATCH 4/4] Update
 packages/reporting/src/webrequest-pipeline/page-store.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Philipp Claßen 
---
 packages/reporting/src/webrequest-pipeline/page-store.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/reporting/src/webrequest-pipeline/page-store.js b/packages/reporting/src/webrequest-pipeline/page-store.js
index 119cbcf2..73062bdd 100644
--- a/packages/reporting/src/webrequest-pipeline/page-store.js
+++ b/packages/reporting/src/webrequest-pipeline/page-store.js
@@ -144,7 +144,7 @@ export default class PageStore {
     const { frameId, tabId, url, timeStamp } = details;
     const tabContext = this.tabs.get(tabId);
     if (frameId === 0) {
-      // ignore dupilcated onBeforeNavigate https://bugzilla.mozilla.org/show_bug.cgi?id=1732564
+      // ignore duplicated onBeforeNavigate https://bugzilla.mozilla.org/show_bug.cgi?id=1732564
       if (
         tabContext &&
         tabContext.id === tabId &&