From a7cd7e0eea1511908ab3178ca7ebc1f08799d550 Mon Sep 17 00:00:00 2001 From: benjackwhite Date: Thu, 10 Aug 2023 08:16:19 +0000 Subject: [PATCH 01/13] chore: Bump version to 1.76.0 --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2a65da35..6cdbbf01e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.76.0 - 2023-08-10 + +- Fixed up tests to cover all cases (#770) + ## 1.75.4 - 2023-08-09 - feat: remove old UUID code (#755) diff --git a/package.json b/package.json index f88ad510f..ed8ce636b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-js", - "version": "1.75.4", + "version": "1.76.0", "description": "Posthog-js allows you to automatically capture usage and send events to PostHog.", "repository": "https://github.com/PostHog/posthog-js", "author": "hey@posthog.com", From e69e4881097a4031219a43c867f0be379d2707de Mon Sep 17 00:00:00 2001 From: David Newell Date: Mon, 14 Aug 2023 16:20:25 +0100 Subject: [PATCH 02/13] chore: add media examples to playground (#771) --- playground/nextjs/pages/index.tsx | 1 + playground/nextjs/pages/media.tsx | 29 +++++++++++++++++++++++++++++ playground/nextjs/yarn.lock | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 playground/nextjs/pages/media.tsx diff --git a/playground/nextjs/pages/index.tsx b/playground/nextjs/pages/index.tsx index 94e98f2ee..7f209b60e 100644 --- a/playground/nextjs/pages/index.tsx +++ b/playground/nextjs/pages/index.tsx @@ -54,6 +54,7 @@ export default function Home() {
Animations Iframe + Media

Feature flag response: {JSON.stringify(result)}

diff --git a/playground/nextjs/pages/media.tsx b/playground/nextjs/pages/media.tsx new file mode 100644 index 000000000..963963925 --- /dev/null +++ b/playground/nextjs/pages/media.tsx @@ -0,0 +1,29 @@ +import Head from 'next/head' + +export default function Home() { + return ( + <> + + PostHog + + +
+

Video

+

Useful testing for Replay handling video elements

+
+

Video

+ +
+ +
+

Audio

+ +
+
+ + ) +} diff --git a/playground/nextjs/yarn.lock b/playground/nextjs/yarn.lock index ea12b3937..fbcf80bf6 100644 --- a/playground/nextjs/yarn.lock +++ b/playground/nextjs/yarn.lock @@ -1920,7 +1920,7 @@ scheduler@^0.23.0: dependencies: loose-envify "^1.1.0" -semver@^6.3.0: +semver@^6.3.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" From 229a94d2d303f3e214711f18c6b3672c4f83ad0b Mon Sep 17 00:00:00 2001 From: Robbie Date: Fri, 18 Aug 2023 08:59:37 +0100 Subject: [PATCH 03/13] style: Tighten eslint rules (#775) * Tighten eslint rules * Bump node in github actions to 18 --- .eslintrc.js | 62 ++++--- .github/workflows/cd.yml | 6 +- .github/workflows/library-ci.yml | 9 +- .github/workflows/react.yml | 5 +- .github/workflows/ssr-es-check.yml | 6 +- .github/workflows/testcafe.yml | 6 +- package.json | 5 +- src/__tests__/extensions/toolbar.js | 2 - src/__tests__/posthog-core.js | 12 +- src/extensions/sessionrecording-utils.ts | 2 +- src/utils.ts | 1 - yarn.lock | 220 +++++++++++++++++------ 12 files changed, 227 insertions(+), 109 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index afcf3de4d..f901c4877 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,28 +1,44 @@ +/*eslint-env node */ + +const rules = { + 'prettier/prettier': 'error', + 'prefer-spread': 'off', + '@typescript-eslint/no-empty-function': 'off', + '@typescript-eslint/no-this-alias': 'off', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': ['error'], + 'no-prototype-builtins': 'off', + 'no-empty': 'off', +} + +const extend = [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + 'plugin:react/recommended', + 'plugin:react-hooks/recommended', + 'prettier', + 'plugin:compat/recommended', +] + module.exports = { env: { browser: true, es6: true, + 'jest/globals': true, + }, + globals: { + given: 'readonly', + global: 'readonly', + Buffer: 'readonly', }, parser: '@typescript-eslint/parser', parserOptions: { ecmaVersion: 2018, sourceType: 'module', }, - plugins: ['prettier', '@typescript-eslint', 'eslint-plugin-react', 'eslint-plugin-react-hooks'], - extends: [ - 'plugin:@typescript-eslint/recommended', - 'plugin:react/recommended', - 'plugin:react-hooks/recommended', - 'prettier', - 'plugin:compat/recommended', - ], - rules: { - 'prettier/prettier': 'error', - 'prefer-spread': 'off', - '@typescript-eslint/no-empty-function': 'off', - '@typescript-eslint/no-this-alias': 'off', - '@typescript-eslint/no-explicit-any': 'off', - }, + plugins: ['prettier', '@typescript-eslint', 'eslint-plugin-react', 'eslint-plugin-react-hooks', 'jest'], + extends: extend, + rules, settings: { react: { version: '17.0', @@ -34,19 +50,9 @@ module.exports = { // the same set of config as in the root // but excluding the 'plugin:compat/recommended' rule // we don't mind using the latest features in our tests - extends: [ - 'plugin:@typescript-eslint/recommended', - 'plugin:react/recommended', - 'plugin:react-hooks/recommended', - 'prettier', - ], - rules: { - 'prettier/prettier': 'error', - 'prefer-spread': 'off', - '@typescript-eslint/no-empty-function': 'off', - '@typescript-eslint/no-this-alias': 'off', - '@typescript-eslint/no-explicit-any': 'off', - }, + extends: extend.filter((s) => s !== 'plugin:compat/recommended'), + rules, }, ], + root: true, } diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index e3b895251..ce061f90d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -37,10 +37,10 @@ jobs: fetch-depth: 0 token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} - - name: Set up Node 14 - uses: actions/setup-node@v2 + - name: Set up Node 18 + uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 registry-url: https://registry.npmjs.org - name: Install dependencies diff --git a/.github/workflows/library-ci.yml b/.github/workflows/library-ci.yml index e1e6a23ff..8de989a55 100644 --- a/.github/workflows/library-ci.yml +++ b/.github/workflows/library-ci.yml @@ -17,7 +17,7 @@ jobs: # Install Node.js - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 cache: 'yarn' # Install your dependencies @@ -33,7 +33,8 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 + cache: 'yarn' - run: yarn install && yarn build @@ -50,7 +51,7 @@ jobs: # Install Node.js - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 cache: 'yarn' # Install your dependencies @@ -68,7 +69,7 @@ jobs: # Install Node.js - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 cache: 'yarn' # Install your dependencies diff --git a/.github/workflows/react.yml b/.github/workflows/react.yml index 65faceb8e..76db9d7bf 100644 --- a/.github/workflows/react.yml +++ b/.github/workflows/react.yml @@ -9,7 +9,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 + cache: 'yarn' - run: yarn && yarn build && cd react && yarn test diff --git a/.github/workflows/ssr-es-check.yml b/.github/workflows/ssr-es-check.yml index ddb405843..51a21c24d 100644 --- a/.github/workflows/ssr-es-check.yml +++ b/.github/workflows/ssr-es-check.yml @@ -9,9 +9,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 + cache: 'yarn' + - run: yarn install && yarn build diff --git a/.github/workflows/testcafe.yml b/.github/workflows/testcafe.yml index 20366768e..6258ae869 100644 --- a/.github/workflows/testcafe.yml +++ b/.github/workflows/testcafe.yml @@ -32,10 +32,10 @@ jobs: - name: Check out posthog-js uses: actions/checkout@v2 - - name: Set up Node 14 - uses: actions/setup-node@v2 + - name: Set up Node 18 + uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 - name: Serve static files run: python -m http.server 8080 & diff --git a/package.json b/package.json index ed8ce636b..0c5f7d52d 100644 --- a/package.json +++ b/package.json @@ -46,14 +46,15 @@ "@types/jest": "^29.5.1", "@types/react-dom": "^18.0.10", "@types/uuid": "^9.0.1", - "@typescript-eslint/eslint-plugin": "^5.30.7", - "@typescript-eslint/parser": "^5.30.7", + "@typescript-eslint/eslint-plugin": "^6.4.0", + "@typescript-eslint/parser": "^6.4.0", "babel-eslint": "10.1.0", "babel-jest": "^26.6.3", "cypress": "10.3.1", "eslint": "8.20.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-compat": "^4.1.4", + "eslint-plugin-jest": "^27.2.3", "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react": "^7.30.1", "eslint-plugin-react-hooks": "^4.6.0", diff --git a/src/__tests__/extensions/toolbar.js b/src/__tests__/extensions/toolbar.js index a6bff5057..b18a4f01e 100644 --- a/src/__tests__/extensions/toolbar.js +++ b/src/__tests__/extensions/toolbar.js @@ -98,8 +98,6 @@ describe('Toolbar', () => { }) it('should initialize the toolbar when there are editor params in the session', () => { - given('storedEditorParams', () => JSON.stringify(toolbarParams)) - given.subject() expect(given.toolbar.loadToolbar).toHaveBeenCalledWith({ ...given.toolbarParams, diff --git a/src/__tests__/posthog-core.js b/src/__tests__/posthog-core.js index 3d211759d..967ed3c30 100644 --- a/src/__tests__/posthog-core.js +++ b/src/__tests__/posthog-core.js @@ -599,7 +599,7 @@ describe('init()', () => { delete window.rrwebRecord window.rrwebRecord = 'is possible' given.subject() - expect(given.lib.__loaded_recorder_version).toMatch(new RegExp(`^1\.?`)) // start with 1.?.? + expect(given.lib.__loaded_recorder_version).toMatch(/^1\./) // start with 1.?.? }) it('set __loaded_recorder_version flag to v1 if recording script has been included', () => { @@ -611,7 +611,7 @@ describe('init()', () => { delete window.rrwebRecord window.rrwebRecord = 'is possible' given.subject() - expect(given.lib.__loaded_recorder_version).toMatch(new RegExp(`^2\.?`)) // start with 2.?.? + expect(given.lib.__loaded_recorder_version).toMatch(/^2\./) // start with 2.?.? }) it('does not load autocapture, feature flags, toolbar, session recording or compression', () => { @@ -661,8 +661,8 @@ describe('init()', () => { expect(given.lib.register_once).toHaveBeenCalledWith( { - $device_id: truth((val) => val.match(/^[0-9a-f\-]+$/)), - distinct_id: truth((val) => val.match(/^[0-9a-f\-]+$/)), + $device_id: truth((val) => val.match(/^[0-9a-f-]+$/)), + distinct_id: truth((val) => val.match(/^[0-9a-f-]+$/)), }, '' ) @@ -686,8 +686,8 @@ describe('init()', () => { expect(given.lib.register_once).toHaveBeenCalledWith( { - $device_id: truth((val) => val.match(/^custom\-[0-9a-f]+/)), - distinct_id: truth((val) => val.match(/^custom\-[0-9a-f]+/)), + $device_id: truth((val) => val.match(/^custom-[0-9a-f]+/)), + distinct_id: truth((val) => val.match(/^custom-[0-9a-f]+/)), }, '' ) diff --git a/src/extensions/sessionrecording-utils.ts b/src/extensions/sessionrecording-utils.ts index 3546c3c3a..64378c751 100644 --- a/src/extensions/sessionrecording-utils.ts +++ b/src/extensions/sessionrecording-utils.ts @@ -79,7 +79,7 @@ export function ensureMaxMessageSize(data: eventWithTime): { event: eventWithTim // 1) Checks if the pattern starts with 'data:' (potentially, not at the start of the string) // 2) Extracts the mime type of the data uri in the first group // 3) Determines when the data URI ends.Depending on if it's used in the src tag or css, it can end with a ) or " - const dataURIRegex = /data:([\w\/\-\.]+);(\w+),([^)"]*)/gim + const dataURIRegex = /data:([\w/\-.]+);(\w+),([^)"]*)/gim const matches = stringifiedData.matchAll(dataURIRegex) for (const match of matches) { if (match[1].toLocaleLowerCase().slice(0, 6) === 'image/') { diff --git a/src/utils.ts b/src/utils.ts index 24b05a806..c89f5935f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,5 @@ import Config from './config' import { Breaker, EventHandler, Properties } from './types' -import { uuidv7 } from './uuidv7' /* * Saved references to long variable names, so that closure compiler can diff --git a/yarn.lock b/yarn.lock index 557dba646..49ca3c51f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2706,6 +2706,18 @@ debug "^3.1.0" lodash.once "^4.1.1" +"@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": + version "4.4.0" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" + integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== + dependencies: + eslint-visitor-keys "^3.3.0" + +"@eslint-community/regexpp@^4.5.1": + version "4.6.2" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.6.2.tgz#1816b5f6948029c5eaacb0703b850ee0cb37d8f8" + integrity sha512-pPTNuaAG3QMH+buKyBIGJs3g/S5y0caxw0ygM3YyE6yJFySwiGGSzA+mM3KJ8QQvzeLh3blwgSonkFjgQdxzMw== + "@eslint/eslintrc@^1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.0.tgz#29f92c30bb3e771e4a2048c95fa6855392dfac4f" @@ -3659,6 +3671,11 @@ resolved "https://registry.yarnpkg.com/@types/js-levenshtein/-/js-levenshtein-1.1.1.tgz#ba05426a43f9e4e30b631941e0aa17bf0c890ed5" integrity sha512-qC4bCqYGy1y/NP7dDVr7KJarn+PbX1nSpwA7JXdu0HxT3QYjO8MJ+cntENtHFVy2dRAyBV23OZ6MxsW1AM1L8g== +"@types/json-schema@^7.0.12": + version "7.0.12" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" + integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" @@ -3737,6 +3754,11 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== +"@types/semver@^7.3.12", "@types/semver@^7.5.0": + version "7.5.0" + resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.0.tgz#591c1ce3a702c45ee15f47a42ade72c2fd78978a" + integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw== + "@types/set-cookie-parser@^2.4.0": version "2.4.2" resolved "https://registry.yarnpkg.com/@types/set-cookie-parser/-/set-cookie-parser-2.4.2.tgz#b6a955219b54151bfebd4521170723df5e13caad" @@ -3797,86 +3819,139 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@^5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.30.7.tgz#1621dabc1ae4084310e19e9efc80dfdbb97e7493" - integrity sha512-l4L6Do+tfeM2OK0GJsU7TUcM/1oN/N25xHm3Jb4z3OiDU4Lj8dIuxX9LpVMS9riSXQs42D1ieX7b85/r16H9Fw== +"@typescript-eslint/eslint-plugin@^6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.4.0.tgz#53428b616f7d80fe879f45a08f11cc0f0b62cf13" + integrity sha512-62o2Hmc7Gs3p8SLfbXcipjWAa6qk2wZGChXG2JbBtYpwSRmti/9KHLqfbLs9uDigOexG+3PaQ9G2g3201FWLKg== dependencies: - "@typescript-eslint/scope-manager" "5.30.7" - "@typescript-eslint/type-utils" "5.30.7" - "@typescript-eslint/utils" "5.30.7" + "@eslint-community/regexpp" "^4.5.1" + "@typescript-eslint/scope-manager" "6.4.0" + "@typescript-eslint/type-utils" "6.4.0" + "@typescript-eslint/utils" "6.4.0" + "@typescript-eslint/visitor-keys" "6.4.0" + debug "^4.3.4" + graphemer "^1.4.0" + ignore "^5.2.4" + natural-compare "^1.4.0" + semver "^7.5.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/parser@^6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.4.0.tgz#47e7c6e22ff1248e8675d95f488890484de67600" + integrity sha512-I1Ah1irl033uxjxO9Xql7+biL3YD7w9IU8zF+xlzD/YxY6a4b7DYA08PXUUCbm2sEljwJF6ERFy2kTGAGcNilg== + dependencies: + "@typescript-eslint/scope-manager" "6.4.0" + "@typescript-eslint/types" "6.4.0" + "@typescript-eslint/typescript-estree" "6.4.0" + "@typescript-eslint/visitor-keys" "6.4.0" debug "^4.3.4" - functional-red-black-tree "^1.0.1" - ignore "^5.2.0" - regexpp "^3.2.0" - semver "^7.3.7" - tsutils "^3.21.0" -"@typescript-eslint/parser@^5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.30.7.tgz#99d09729392aec9e64b1de45cd63cb81a4ddd980" - integrity sha512-Rg5xwznHWWSy7v2o0cdho6n+xLhK2gntImp0rJroVVFkcYFYQ8C8UJTSuTw/3CnExBmPjycjmUJkxVmjXsld6A== +"@typescript-eslint/scope-manager@5.62.0": + version "5.62.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.62.0.tgz#d9457ccc6a0b8d6b37d0eb252a23022478c5460c" + integrity sha512-VXuvVvZeQCQb5Zgf4HAxc04q5j+WrNAtNh9OwCsCgpKqESMTu3tF/jhZ3xG6T4NZwWl65Bg8KuS2uEvhSfLl0w== dependencies: - "@typescript-eslint/scope-manager" "5.30.7" - "@typescript-eslint/types" "5.30.7" - "@typescript-eslint/typescript-estree" "5.30.7" - debug "^4.3.4" + "@typescript-eslint/types" "5.62.0" + "@typescript-eslint/visitor-keys" "5.62.0" -"@typescript-eslint/scope-manager@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.30.7.tgz#8269a931ef1e5ae68b5eb80743cc515c4ffe3dd7" - integrity sha512-7BM1bwvdF1UUvt+b9smhqdc/eniOnCKxQT/kj3oXtj3LqnTWCAM0qHRHfyzCzhEfWX0zrW7KqXXeE4DlchZBKw== +"@typescript-eslint/scope-manager@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.4.0.tgz#3048e4262ba3eafa4e2e69b08912d9037ec646ae" + integrity sha512-TUS7vaKkPWDVvl7GDNHFQMsMruD+zhkd3SdVW0d7b+7Zo+bd/hXJQ8nsiUZMi1jloWo6c9qt3B7Sqo+flC1nig== dependencies: - "@typescript-eslint/types" "5.30.7" - "@typescript-eslint/visitor-keys" "5.30.7" + "@typescript-eslint/types" "6.4.0" + "@typescript-eslint/visitor-keys" "6.4.0" -"@typescript-eslint/type-utils@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.30.7.tgz#5693dc3db6f313f302764282d614cfdbc8a9fcfd" - integrity sha512-nD5qAE2aJX/YLyKMvOU5jvJyku4QN5XBVsoTynFrjQZaDgDV6i7QHFiYCx10wvn7hFvfuqIRNBtsgaLe0DbWhw== +"@typescript-eslint/type-utils@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.4.0.tgz#c8ac92716ed6a9d5443aa3e342910355b0796ba0" + integrity sha512-TvqrUFFyGY0cX3WgDHcdl2/mMCWCDv/0thTtx/ODMY1QhEiyFtv/OlLaNIiYLwRpAxAtOLOY9SUf1H3Q3dlwAg== dependencies: - "@typescript-eslint/utils" "5.30.7" + "@typescript-eslint/typescript-estree" "6.4.0" + "@typescript-eslint/utils" "6.4.0" debug "^4.3.4" - tsutils "^3.21.0" + ts-api-utils "^1.0.1" + +"@typescript-eslint/types@5.62.0": + version "5.62.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.62.0.tgz#258607e60effa309f067608931c3df6fed41fd2f" + integrity sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ== -"@typescript-eslint/types@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.30.7.tgz#18331487cc92d0f1fb1a6f580c8ec832528079d0" - integrity sha512-ocVkETUs82+U+HowkovV6uxf1AnVRKCmDRNUBUUo46/5SQv1owC/EBFkiu4MOHeZqhKz2ktZ3kvJJ1uFqQ8QPg== +"@typescript-eslint/types@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.4.0.tgz#5b109a59a805f0d8d375895e42d9e5f0037f66ee" + integrity sha512-+FV9kVFrS7w78YtzkIsNSoYsnOtrYVnKWSTVXoL1761CsCRv5wpDOINgsXpxD67YCLZtVQekDDyaxfjVWUJmmg== -"@typescript-eslint/typescript-estree@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.30.7.tgz#05da9f1b281985bfedcf62349847f8d168eecc07" - integrity sha512-tNslqXI1ZdmXXrHER83TJ8OTYl4epUzJC0aj2i4DMDT4iU+UqLT3EJeGQvJ17BMbm31x5scSwo3hPM0nqQ1AEA== +"@typescript-eslint/typescript-estree@5.62.0": + version "5.62.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.62.0.tgz#7d17794b77fabcac615d6a48fb143330d962eb9b" + integrity sha512-CmcQ6uY7b9y694lKdRB8FEel7JbU/40iSAPomu++SjLMntB+2Leay2LO6i8VnJk58MtE9/nQSFIH6jpyRWyYzA== dependencies: - "@typescript-eslint/types" "5.30.7" - "@typescript-eslint/visitor-keys" "5.30.7" + "@typescript-eslint/types" "5.62.0" + "@typescript-eslint/visitor-keys" "5.62.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" semver "^7.3.7" tsutils "^3.21.0" -"@typescript-eslint/utils@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.30.7.tgz#7135be070349e9f7caa262b0ca59dc96123351bb" - integrity sha512-Z3pHdbFw+ftZiGUnm1GZhkJgVqsDL5CYW2yj+TB2mfXDFOMqtbzQi2dNJIyPqPbx9mv2kUxS1gU+r2gKlKi1rQ== +"@typescript-eslint/typescript-estree@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.4.0.tgz#3c58d20632db93fec3d6ab902acbedf593d37276" + integrity sha512-iDPJArf/K2sxvjOR6skeUCNgHR/tCQXBsa+ee1/clRKr3olZjZ/dSkXPZjG6YkPtnW6p5D1egeEPMCW6Gn4yLA== dependencies: + "@typescript-eslint/types" "6.4.0" + "@typescript-eslint/visitor-keys" "6.4.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + semver "^7.5.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/utils@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.4.0.tgz#23e996b693603c5924b1fbb733cc73196256baa5" + integrity sha512-BvvwryBQpECPGo8PwF/y/q+yacg8Hn/2XS+DqL/oRsOPK+RPt29h5Ui5dqOKHDlbXrAeHUTnyG3wZA0KTDxRZw== + dependencies: + "@eslint-community/eslint-utils" "^4.4.0" + "@types/json-schema" "^7.0.12" + "@types/semver" "^7.5.0" + "@typescript-eslint/scope-manager" "6.4.0" + "@typescript-eslint/types" "6.4.0" + "@typescript-eslint/typescript-estree" "6.4.0" + semver "^7.5.4" + +"@typescript-eslint/utils@^5.10.0": + version "5.62.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.62.0.tgz#141e809c71636e4a75daa39faed2fb5f4b10df86" + integrity sha512-n8oxjeb5aIbPFEtmQxQYOLI0i9n5ySBEY/ZEHHZqKQSFnxio1rv6dthascc9dLuwrL0RC5mPCxB7vnAVGAYWAQ== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" "@types/json-schema" "^7.0.9" - "@typescript-eslint/scope-manager" "5.30.7" - "@typescript-eslint/types" "5.30.7" - "@typescript-eslint/typescript-estree" "5.30.7" + "@types/semver" "^7.3.12" + "@typescript-eslint/scope-manager" "5.62.0" + "@typescript-eslint/types" "5.62.0" + "@typescript-eslint/typescript-estree" "5.62.0" eslint-scope "^5.1.1" - eslint-utils "^3.0.0" + semver "^7.3.7" -"@typescript-eslint/visitor-keys@5.30.7": - version "5.30.7" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.30.7.tgz#c093abae75b4fd822bfbad9fc337f38a7a14909a" - integrity sha512-KrRXf8nnjvcpxDFOKej4xkD7657+PClJs5cJVSG7NNoCNnjEdc46juNAQt7AyuWctuCgs6mVRc1xGctEqrjxWw== +"@typescript-eslint/visitor-keys@5.62.0": + version "5.62.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.62.0.tgz#2174011917ce582875954ffe2f6912d5931e353e" + integrity sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw== dependencies: - "@typescript-eslint/types" "5.30.7" + "@typescript-eslint/types" "5.62.0" eslint-visitor-keys "^3.3.0" +"@typescript-eslint/visitor-keys@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.4.0.tgz#96a426cdb1add28274abd7a34aefe27f8b7d51ef" + integrity sha512-yJSfyT+uJm+JRDWYRYdCm2i+pmvXJSMtPR9Cq5/XQs4QIgNoLcoRtDdzsLbLsFM/c6um6ohQkg/MLxWvoIndJA== + dependencies: + "@typescript-eslint/types" "6.4.0" + eslint-visitor-keys "^3.4.1" + "@xmldom/xmldom@^0.8.3": version "0.8.7" resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.7.tgz#8b1e39c547013941974d83ad5e9cf5042071a9a0" @@ -5766,6 +5841,13 @@ eslint-plugin-compat@^4.1.4: lodash.memoize "4.1.2" semver "7.3.8" +eslint-plugin-jest@^27.2.3: + version "27.2.3" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-27.2.3.tgz#6f8a4bb2ca82c0c5d481d1b3be256ab001f5a3ec" + integrity sha512-sRLlSCpICzWuje66Gl9zvdF6mwD5X86I4u55hJyFBsxYOsBCmT5+kSUjf+fkFWVMMgpzNEupjW8WzUqi83hJAQ== + dependencies: + "@typescript-eslint/utils" "^5.10.0" + eslint-plugin-prettier@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz#651cbb88b1dab98bfd42f017a12fa6b2d993f94b" @@ -5836,6 +5918,11 @@ eslint-visitor-keys@^3.3.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== +eslint-visitor-keys@^3.4.1: + version "3.4.3" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" + integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== + eslint@8.20.0: version "8.20.0" resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.20.0.tgz#048ac56aa18529967da8354a478be4ec0a2bc81b" @@ -6759,6 +6846,11 @@ graceful-fs@^4.2.9: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== + graphlib@^2.1.5: version "2.1.8" resolved "https://registry.yarnpkg.com/graphlib/-/graphlib-2.1.8.tgz#5761d414737870084c92ec7b5dbcb0592c9d35da" @@ -6997,6 +7089,11 @@ ignore@^5.2.0: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== +ignore@^5.2.4: + version "5.2.4" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" + integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== + image-q@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/image-q/-/image-q-1.1.1.tgz#fc84099664460b90ca862d9300b6bfbbbfbf8056" @@ -9734,6 +9831,7 @@ posix-character-classes@^0.1.0: "posthog-js@link:.": version "0.0.0" + uid "" prelude-ls@^1.2.1: version "1.2.1" @@ -10532,6 +10630,13 @@ semver@^7.3.7: dependencies: lru-cache "^6.0.0" +semver@^7.5.4: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" @@ -11515,6 +11620,11 @@ truncate-utf8-bytes@^1.0.0: dependencies: utf8-byte-length "^1.0.1" +ts-api-utils@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.1.tgz#8144e811d44c749cd65b2da305a032510774452d" + integrity sha512-lC/RGlPmwdrIBFTX59wwNzqh7aR2otPNPR/5brHZm/XKFYKsfqxihXUe9pU3JI+3vGkl+vyCoNNnPhJn3aLK1A== + tslib@^1.8.1, tslib@^1.9.0: version "1.13.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" From 9a02e5d6c0d286df6754c72d61281609f519cb89 Mon Sep 17 00:00:00 2001 From: Robbie Date: Fri, 18 Aug 2023 09:13:45 +0100 Subject: [PATCH 04/13] feat: Add previous page properties to page events (#773) * Add previous page properties to page events * Add scroll measuring * Add tests for scroll position * Remove duplicate test * Update src/page-view.ts Co-authored-by: Ben White * Remove passing in the window object * Put pageview stats behind __preview_measure_pageview_stats * Kill page view ids * Delete unused interface/import --------- Co-authored-by: Ben White --- playground/nextjs/.eslintrc.json | 3 - playground/nextjs/pages/_app.tsx | 3 +- playground/nextjs/pages/index.tsx | 4 +- playground/nextjs/pages/long.tsx | 34 +++++++ src/__tests__/page-view-id.js | 25 ----- src/__tests__/page-view.ts | 103 +++++++++++++++++++ src/page-view-id.ts | 24 ----- src/page-view.ts | 158 ++++++++++++++++++++++++++++++ src/posthog-core.ts | 19 ++-- src/types.ts | 1 + 10 files changed, 313 insertions(+), 61 deletions(-) delete mode 100644 playground/nextjs/.eslintrc.json create mode 100644 playground/nextjs/pages/long.tsx delete mode 100644 src/__tests__/page-view-id.js create mode 100644 src/__tests__/page-view.ts delete mode 100644 src/page-view-id.ts create mode 100644 src/page-view.ts diff --git a/playground/nextjs/.eslintrc.json b/playground/nextjs/.eslintrc.json deleted file mode 100644 index bffb357a7..000000000 --- a/playground/nextjs/.eslintrc.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "next/core-web-vitals" -} diff --git a/playground/nextjs/pages/_app.tsx b/playground/nextjs/pages/_app.tsx index b91a43e2a..32764f904 100644 --- a/playground/nextjs/pages/_app.tsx +++ b/playground/nextjs/pages/_app.tsx @@ -1,6 +1,6 @@ import '@/styles/globals.css' -import { useEffect } from 'react' +import React, { useEffect } from 'react' import type { AppProps } from 'next/app' import { useRouter } from 'next/router' @@ -13,6 +13,7 @@ if (typeof window !== 'undefined') { session_recording: { recordCrossOriginIframes: true, }, + debug: true, }) ;(window as any).posthog = posthog } diff --git a/playground/nextjs/pages/index.tsx b/playground/nextjs/pages/index.tsx index 7f209b60e..a9164fac5 100644 --- a/playground/nextjs/pages/index.tsx +++ b/playground/nextjs/pages/index.tsx @@ -1,7 +1,6 @@ import Head from 'next/head' import { useFeatureFlagEnabled, usePostHog } from 'posthog-js/react' -import { useEffect, useState } from 'react' -import { Player, Controls } from '@lottiefiles/react-lottie-player' +import React, { useEffect, useState } from 'react' import Link from 'next/link' export default function Home() { @@ -55,6 +54,7 @@ export default function Home() { Animations Iframe Media + Long

Feature flag response: {JSON.stringify(result)}

diff --git a/playground/nextjs/pages/long.tsx b/playground/nextjs/pages/long.tsx new file mode 100644 index 000000000..a136d9604 --- /dev/null +++ b/playground/nextjs/pages/long.tsx @@ -0,0 +1,34 @@ +import Head from 'next/head' +import Link from 'next/link' +import React from 'react' + +export default function Home() { + return ( + <> + + PostHog + + +
+

A long page

+
+ Home +
+ + {Array.from({ length: 100 }, (_, i) => ( +

+ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut + labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco + laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in + voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat + non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +

+ ))} + +
+ Home +
+
+ + ) +} diff --git a/src/__tests__/page-view-id.js b/src/__tests__/page-view-id.js deleted file mode 100644 index 21e44d47b..000000000 --- a/src/__tests__/page-view-id.js +++ /dev/null @@ -1,25 +0,0 @@ -import { PageViewIdManager } from '../page-view-id' -import { uuidv7 } from '../uuidv7' -jest.mock('../uuidv7') - -describe('PageView ID manager', () => { - given('pageViewIdManager', () => new PageViewIdManager()) - - beforeEach(() => { - uuidv7.mockReturnValue('subsequentUUIDs').mockReturnValueOnce('firstUUID').mockReturnValueOnce('secondUUID') - }) - - it('generates a page view id and resets page view id', () => { - expect(given.pageViewIdManager.getPageViewId()).toEqual('firstUUID') - - // First pageview should NOT rotate the UUID - given.pageViewIdManager.onPageview() - expect(given.pageViewIdManager.getPageViewId()).toEqual('firstUUID') - - given.pageViewIdManager.onPageview() - expect(given.pageViewIdManager.getPageViewId()).toEqual('secondUUID') - - given.pageViewIdManager.onPageview() - expect(given.pageViewIdManager.getPageViewId()).toEqual('subsequentUUIDs') - }) -}) diff --git a/src/__tests__/page-view.ts b/src/__tests__/page-view.ts new file mode 100644 index 000000000..14d8ac798 --- /dev/null +++ b/src/__tests__/page-view.ts @@ -0,0 +1,103 @@ +import { PageViewManager } from '../page-view' + +const mockWindowGetter = jest.fn() +jest.mock('../utils', () => ({ + ...jest.requireActual('../utils'), + get window() { + return mockWindowGetter() + }, +})) + +describe('PageView ID manager', () => { + describe('doPageView', () => { + beforeEach(() => { + mockWindowGetter.mockReturnValue({ + location: { + pathname: '/pathname', + }, + scrollY: 0, + document: { + documentElement: { + clientHeight: 0, + scrollHeight: 0, + }, + }, + }) + }) + + it('includes scroll position properties for a partially scrolled long page', () => { + // note that this means that the user has scrolled 2/3rds of the way down the scrollable area, and seen + // 3/4 of the content + mockWindowGetter.mockReturnValue({ + location: { + pathname: '/pathname', + }, + scrollY: 2000, // how far down the user has scrolled + document: { + documentElement: { + clientHeight: 1000, // how tall the window is + scrollHeight: 4000, // how tall the page content is + }, + }, + }) + + const pageViewIdManager = new PageViewManager() + pageViewIdManager.doPageView() + + // force the manager to update the scroll data by calling an internal method + pageViewIdManager._updateScrollData() + + const secondPageView = pageViewIdManager.doPageView() + expect(secondPageView.$prev_pageview_last_scroll).toEqual(2000) + expect(secondPageView.$prev_pageview_last_scroll_percentage).toBeCloseTo(2 / 3) + expect(secondPageView.$prev_pageview_max_scroll).toEqual(2000) + expect(secondPageView.$prev_pageview_max_scroll_percentage).toBeCloseTo(2 / 3) + expect(secondPageView.$prev_pageview_last_content).toEqual(3000) + expect(secondPageView.$prev_pageview_last_content_percentage).toBeCloseTo(3 / 4) + expect(secondPageView.$prev_pageview_max_content).toEqual(3000) + expect(secondPageView.$prev_pageview_max_content_percentage).toBeCloseTo(3 / 4) + }) + + it('includes scroll position properties for a short page', () => { + mockWindowGetter.mockReturnValue({ + location: { + pathname: '/pathname', + }, + scrollY: 0, + document: { + documentElement: { + clientHeight: 1000, // how tall the window is + scrollHeight: 500, // how tall the page content is + }, + }, + }) + + const pageViewIdManager = new PageViewManager() + pageViewIdManager.doPageView() + + // force the manager to update the scroll data by calling an internal method + pageViewIdManager._updateScrollData() + + const secondPageView = pageViewIdManager.doPageView() + expect(secondPageView.$prev_pageview_last_scroll).toEqual(0) + expect(secondPageView.$prev_pageview_last_scroll_percentage).toEqual(1) + expect(secondPageView.$prev_pageview_max_scroll).toEqual(0) + expect(secondPageView.$prev_pageview_max_scroll_percentage).toEqual(1) + expect(secondPageView.$prev_pageview_last_content).toEqual(1000) + expect(secondPageView.$prev_pageview_last_content_percentage).toEqual(1) + expect(secondPageView.$prev_pageview_max_content).toEqual(1000) + expect(secondPageView.$prev_pageview_max_content_percentage).toEqual(1) + }) + + it('can handle scroll updates before doPageView is called', () => { + const pageViewIdManager = new PageViewManager() + + pageViewIdManager._updateScrollData() + const firstPageView = pageViewIdManager.doPageView() + expect(firstPageView.$prev_pageview_last_scroll).toBeUndefined() + + const secondPageView = pageViewIdManager.doPageView() + expect(secondPageView.$prev_pageview_last_scroll).toBeDefined() + }) + }) +}) diff --git a/src/page-view-id.ts b/src/page-view-id.ts deleted file mode 100644 index 02d638ace..000000000 --- a/src/page-view-id.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { uuidv7 } from './uuidv7' - -export class PageViewIdManager { - _pageViewId: string | undefined - - _seenFirstPageView = false - - onPageview(): void { - // As the first $pageview event may come after a different event, - // we only reset the ID _after_ the second $pageview event. - if (this._seenFirstPageView) { - this._pageViewId = uuidv7() - } - this._seenFirstPageView = true - } - - getPageViewId(): string { - if (!this._pageViewId) { - this._pageViewId = uuidv7() - } - - return this._pageViewId - } -} diff --git a/src/page-view.ts b/src/page-view.ts new file mode 100644 index 000000000..665e122ab --- /dev/null +++ b/src/page-view.ts @@ -0,0 +1,158 @@ +import * as utils from './utils' + +interface PageViewData { + // scroll is how far down the page the user has scrolled, + // content is how far down the page the user can view content + // (e.g. if the page is 1000 tall, but the user's screen is only 500 tall, + // and they don't scroll at all, then scroll is 0 and content is 500) + maxScrollHeight?: number + maxScrollY?: number + lastScrollY?: number + maxContentHeight?: number + maxContentY?: number + lastContentY?: number +} + +interface ScrollProperties { + $prev_pageview_last_scroll?: number + $prev_pageview_last_scroll_percentage?: number + $prev_pageview_max_scroll?: number + $prev_pageview_max_scroll_percentage?: number + $prev_pageview_last_content?: number + $prev_pageview_last_content_percentage?: number + $prev_pageview_max_content?: number + $prev_pageview_max_content_percentage?: number +} + +export class PageViewManager { + _pageViewData: PageViewData | undefined + _hasSeenPageView = false + + _createPageViewData(): PageViewData { + return {} + } + + doPageView(): ScrollProperties { + let prevPageViewData: PageViewData | undefined + // if there were events created before the first PageView, we would have created a + // pageViewData for them. If this happened, we don't want to create a new pageViewData + if (!this._hasSeenPageView) { + this._hasSeenPageView = true + prevPageViewData = undefined + if (!this._pageViewData) { + this._pageViewData = this._createPageViewData() + } + } else { + prevPageViewData = this._pageViewData + this._pageViewData = this._createPageViewData() + } + + // update the scroll properties for the new page, but wait until the next tick + // of the event loop + setTimeout(this._updateScrollData, 0) + + return this._calculatePrevPageScrollProperties(prevPageViewData) + } + + doPageLeave(): ScrollProperties { + const prevPageViewData = this._pageViewData + return this._calculatePrevPageScrollProperties(prevPageViewData) + } + + _calculatePrevPageScrollProperties(prevPageViewData: PageViewData | undefined): ScrollProperties { + if ( + !prevPageViewData || + prevPageViewData.maxScrollHeight == null || + prevPageViewData.lastScrollY == null || + prevPageViewData.maxScrollY == null || + prevPageViewData.maxContentHeight == null || + prevPageViewData.lastContentY == null || + prevPageViewData.maxContentY == null + ) { + return {} + } + + let { maxScrollHeight, lastScrollY, maxScrollY, maxContentHeight, lastContentY, maxContentY } = prevPageViewData + + // Use ceil, so that e.g. scrolling 999.5px of a 1000px page is considered 100% scrolled + maxScrollHeight = Math.ceil(maxScrollHeight) + lastScrollY = Math.ceil(lastScrollY) + maxScrollY = Math.ceil(maxScrollY) + maxContentHeight = Math.ceil(maxContentHeight) + lastContentY = Math.ceil(lastContentY) + maxContentY = Math.ceil(maxContentY) + + // if the maximum scroll height is near 0, then the percentage is 1 + const lastScrollPercentage = maxScrollHeight <= 1 ? 1 : clamp(lastScrollY / maxScrollHeight, 0, 1) + const maxScrollPercentage = maxScrollHeight <= 1 ? 1 : clamp(maxScrollY / maxScrollHeight, 0, 1) + const lastContentPercentage = maxContentHeight <= 1 ? 1 : clamp(lastContentY / maxContentHeight, 0, 1) + const maxContentPercentage = maxContentHeight <= 1 ? 1 : clamp(maxContentY / maxContentHeight, 0, 1) + + return { + $prev_pageview_last_scroll: lastScrollY, + $prev_pageview_last_scroll_percentage: lastScrollPercentage, + $prev_pageview_max_scroll: maxScrollY, + $prev_pageview_max_scroll_percentage: maxScrollPercentage, + $prev_pageview_last_content: lastContentY, + $prev_pageview_last_content_percentage: lastContentPercentage, + $prev_pageview_max_content: maxContentY, + $prev_pageview_max_content_percentage: maxContentPercentage, + } + } + + _updateScrollData = () => { + if (!this._pageViewData) { + this._pageViewData = this._createPageViewData() + } + const pageViewData = this._pageViewData + + const scrollY = this._scrollY() + const scrollHeight = this._scrollHeight() + const contentY = this._contentY() + const contentHeight = this._contentHeight() + + pageViewData.lastScrollY = scrollY + pageViewData.maxScrollY = Math.max(scrollY, pageViewData.maxScrollY ?? 0) + pageViewData.maxScrollHeight = Math.max(scrollHeight, pageViewData.maxScrollHeight ?? 0) + + pageViewData.lastContentY = contentY + pageViewData.maxContentY = Math.max(contentY, pageViewData.maxContentY ?? 0) + pageViewData.maxContentHeight = Math.max(contentHeight, pageViewData.maxContentHeight ?? 0) + } + + startMeasuringScrollPosition() { + utils.window.addEventListener('scroll', this._updateScrollData) + utils.window.addEventListener('scrollend', this._updateScrollData) + utils.window.addEventListener('resize', this._updateScrollData) + } + + stopMeasuringScrollPosition() { + utils.window.removeEventListener('scroll', this._updateScrollData) + utils.window.removeEventListener('scrollend', this._updateScrollData) + utils.window.removeEventListener('resize', this._updateScrollData) + } + + _scrollHeight(): number { + return Math.max( + 0, + utils.window.document.documentElement.scrollHeight - utils.window.document.documentElement.clientHeight + ) + } + + _scrollY(): number { + return utils.window.scrollY || utils.window.pageYOffset || utils.window.document.documentElement.scrollTop || 0 + } + + _contentHeight(): number { + return utils.window.document.documentElement.scrollHeight || 0 + } + + _contentY(): number { + const clientHeight = utils.window.document.documentElement.clientHeight || 0 + return this._scrollY() + clientHeight + } +} + +function clamp(x: number, min: number, max: number) { + return Math.max(min, Math.min(x, max)) +} diff --git a/src/posthog-core.ts b/src/posthog-core.ts index a995c500f..81826d2f3 100644 --- a/src/posthog-core.ts +++ b/src/posthog-core.ts @@ -53,7 +53,7 @@ import { } from './types' import { SentryIntegration } from './extensions/sentry-integration' import { createSegmentIntegration } from './extensions/segment-integration' -import { PageViewIdManager } from './page-view-id' +import { PageViewManager } from './page-view' import { ExceptionObserver } from './extensions/exceptions/exception-autocapture' import { PostHogSurveys, SurveyCallback } from './posthog-surveys' import { RateLimiter } from './rate-limiter' @@ -215,6 +215,10 @@ const create_phlib = function ( instance.webPerformance = new WebPerformanceObserver(instance) instance.webPerformance.startObservingIfEnabled() + if (instance.get_config('__preview_measure_pageview_stats')) { + instance.pageViewManager.startMeasuringScrollPosition() + } + instance.exceptionAutocapture = new ExceptionObserver(instance) instance.__autocapture = instance.get_config('autocapture') @@ -264,7 +268,7 @@ export class PostHog { rateLimiter: RateLimiter sessionPersistence: PostHogPersistence sessionManager: SessionIdManager - pageViewIdManager: PageViewIdManager + pageViewManager: PageViewManager featureFlags: PostHogFeatureFlags surveys: PostHogSurveys toolbar: Toolbar @@ -307,7 +311,7 @@ export class PostHog { this.featureFlags = new PostHogFeatureFlags(this) this.toolbar = new Toolbar(this) - this.pageViewIdManager = new PageViewIdManager() + this.pageViewManager = new PageViewManager() this.surveys = new PostHogSurveys(this) this.rateLimiter = new RateLimiter() @@ -933,11 +937,14 @@ export class PostHog { properties['$window_id'] = windowId } - if (this.webPerformance?.isEnabled) { + if (this.get_config('__preview_measure_pageview_stats')) { + let performanceProperties: Record = {} if (event_name === '$pageview') { - this.pageViewIdManager.onPageview() + performanceProperties = this.pageViewManager.doPageView() + } else if (event_name === '$pageleave') { + performanceProperties = this.pageViewManager.doPageLeave() } - properties = _extend(properties, { $pageview_id: this.pageViewIdManager.getPageViewId() }) + properties = _extend(properties, performanceProperties) } if (event_name === '$pageview') { diff --git a/src/types.ts b/src/types.ts index d03005567..e39a4f2a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,6 +113,7 @@ export interface PostHogConfig { featureFlagPayloads?: Record } segment?: any + __preview_measure_pageview_stats?: boolean } export interface OptInOutCapturingOptions { From a3acae6af60a91cdd2c57d6db5201ff5378dad86 Mon Sep 17 00:00:00 2001 From: robbie-c Date: Fri, 18 Aug 2023 08:15:06 +0000 Subject: [PATCH 05/13] chore: Bump version to 1.77.0 --- CHANGELOG.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cdbbf01e..6b15f1f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.77.0 - 2023-08-18 + +- feat: Add previous page properties to page events (#773) +- style: Tighten eslint rules (#775) +- chore: add media examples to playground (#771) + ## 1.76.0 - 2023-08-10 - Fixed up tests to cover all cases (#770) diff --git a/package.json b/package.json index 0c5f7d52d..502bf4823 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-js", - "version": "1.76.0", + "version": "1.77.0", "description": "Posthog-js allows you to automatically capture usage and send events to PostHog.", "repository": "https://github.com/PostHog/posthog-js", "author": "hey@posthog.com", From 7bbb61018957aba3cdb502c65d9efa2accae0e06 Mon Sep 17 00:00:00 2001 From: Robbie Date: Mon, 21 Aug 2023 13:24:57 +0100 Subject: [PATCH 06/13] feat: Filter out events from GPTBot (#772) --- src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index c89f5935f..0a786882c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -466,7 +466,7 @@ export const _utf8Encode = function (string: string): string { // sending false capturing data export const _isBlockedUA = function (ua: string): boolean { if ( - /(google web preview|baiduspider|yandexbot|bingbot|googlebot|yahoo! slurp|ahrefsbot|facebookexternalhit|facebookcatalog|applebot|semrushbot|duckduckbot|twitterbot|rogerbot|linkedinbot|mj12bot|sitebulb|bot.htm|bot.php|hubspot|crawler|prerender)/i.test( + /(google web preview|baiduspider|yandexbot|bingbot|googlebot|yahoo! slurp|ahrefsbot|facebookexternalhit|facebookcatalog|applebot|semrushbot|duckduckbot|twitterbot|rogerbot|linkedinbot|mj12bot|sitebulb|bot.htm|bot.php|hubspot|crawler|prerender|gptbot)/i.test( ua ) ) { From 65171a4ccf4a106d135e732b79bdb48e2734e96e Mon Sep 17 00:00:00 2001 From: Robbie Date: Tue, 22 Aug 2023 07:12:07 +0100 Subject: [PATCH 07/13] fix: Mitigate testcafe flakiness (#779) * Pin the testcafe version * Move test file list to .testcaferc.js * Add BROWSERSTACK_FORCE_PROXY * Add BROWSERSTACK_USE_AUTOMATE * Add yarn cache * Add retries * Remove redundant package script for testcafe --- .github/workflows/testcafe.yml | 10 +++++++-- .testcaferc.js | 3 +++ package.json | 7 +++--- scripts/run-testcafe-with-retries.mjs | 31 +++++++++++++++++++++++++++ yarn.lock | 6 +++--- 5 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 .testcaferc.js create mode 100644 scripts/run-testcafe-with-retries.mjs diff --git a/.github/workflows/testcafe.yml b/.github/workflows/testcafe.yml index 6258ae869..bc71caf98 100644 --- a/.github/workflows/testcafe.yml +++ b/.github/workflows/testcafe.yml @@ -8,6 +8,8 @@ env: BROWSERSTACK_DEBUG: 'true' BROWSERSTACK_NETWORK_LOGS: 'true' BROWSERSTACK_CONSOLE: 'info' + BROWSERSTACK_FORCE_PROXY: 'true' + BROWSERSTACK_USE_AUTOMATE: 'true' jobs: browsers: @@ -36,12 +38,16 @@ jobs: uses: actions/setup-node@v3 with: node-version: 18 + cache: 'yarn' - name: Serve static files run: python -m http.server 8080 & + - name: Install node dependencies + run: yarn + - name: Set up posthog-js - run: yarn && yarn build-rollup + run: yarn build-rollup - name: Run ${{ matrix.name }} test - run: npx testcafe "browserstack:${{ matrix.browserstack }}" testcafe/*.spec.js + run: node scripts/run-testcafe-with-retries.mjs --browser "browserstack:${{ matrix.browserstack }}" --attempts 3 diff --git a/.testcaferc.js b/.testcaferc.js new file mode 100644 index 000000000..c8e6ff7ca --- /dev/null +++ b/.testcaferc.js @@ -0,0 +1,3 @@ +module.exports = { + src: './testcafe', +} diff --git a/package.json b/package.json index 502bf4823..db36da4a6 100644 --- a/package.json +++ b/package.json @@ -78,10 +78,11 @@ "rrweb-snapshot": "2.0.0-alpha.8", "rrweb-v1": "npm:rrweb@1.1.3", "sinon": "9.0.2", - "testcafe": "^1.19.0", - "testcafe-browser-provider-browserstack": "^1.14.0", + "testcafe": "1.19.0", + "testcafe-browser-provider-browserstack": "1.14.0", "tslib": "^2.4.0", - "typescript": "^4.7.4" + "typescript": "^4.7.4", + "yargs": "^17.7.2" }, "lint-staged": { "*.{ts,tsx,js,json}": "prettier --write", diff --git a/scripts/run-testcafe-with-retries.mjs b/scripts/run-testcafe-with-retries.mjs new file mode 100644 index 000000000..83687fc75 --- /dev/null +++ b/scripts/run-testcafe-with-retries.mjs @@ -0,0 +1,31 @@ +import yargs from "yargs" + +import {spawnSync} from "child_process" + + +const main = async () => { + const argv= yargs(process.argv).argv + const attempts = argv.attempts || 3 + const browser = argv.browser + if (!browser) { + throw new Error("Missing browser argument") + } + + + for (let i = 0; i < attempts; i++) { + console.log(`Attempt ${i + 1} of a maximum of ${attempts} attempts`) + const result = spawnSync("yarn", ["testcafe", browser], {stdio: "inherit"}) + if (result.status === 0) { + console.log("Test succeeded") + return + } + console.log("Test failed") + } + throw new Error(`Test failed after ${attempts} attempts`) +} + + +main().catch(e => { + console.error(e) + process.exit(1) +}) diff --git a/yarn.lock b/yarn.lock index 49ca3c51f..2e79f3e4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11182,7 +11182,7 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -testcafe-browser-provider-browserstack@^1.14.0: +testcafe-browser-provider-browserstack@1.14.0: version "1.14.0" resolved "https://registry.yarnpkg.com/testcafe-browser-provider-browserstack/-/testcafe-browser-provider-browserstack-1.14.0.tgz#28cc6fd1cc632d13a76bf13c1de11fbdf5fd7956" integrity sha512-pH9R52qbXY0y5QuHYQ4eHafLVcLN77ypslsY21a8a+/Xp7f/Due7tfHBwGws4B7IydjXhcn/g0G39XXYJJf/2Q== @@ -11349,7 +11349,7 @@ testcafe-safe-storage@^1.1.1: resolved "https://registry.yarnpkg.com/testcafe-safe-storage/-/testcafe-safe-storage-1.1.2.tgz#dacfda9a51c77f61f11b13506d4004dd7f27eb73" integrity sha512-6km7D26+KCQGeFlSQ9fVEU7tD8qdioSpqzxU8CCZkd2KzBS0jTFkqaJ54rPaLdd5+wdhgO3+as3LMm4F0EDeYA== -testcafe@^1.19.0: +testcafe@1.19.0: version "1.19.0" resolved "https://registry.yarnpkg.com/testcafe/-/testcafe-1.19.0.tgz#e26bf04a59bae4ad8c17aa43deba12a8ffcb98b3" integrity sha512-HH6Z60N51SPw7WcNFhvbrcV1a6Dri1T0npFmE8BvxQEjsYog2whtdxj01yfaYrE87AZK/ep6lOwrqy0P6WmsfQ== @@ -12212,7 +12212,7 @@ yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@^17.3.1, yargs@^17.5.1: +yargs@^17.3.1, yargs@^17.5.1, yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From a39d310d7a4a645dc7df529aca54e921bfdc7d8a Mon Sep 17 00:00:00 2001 From: Robbie Date: Tue, 22 Aug 2023 07:24:48 +0100 Subject: [PATCH 08/13] feat: Add pathname to prev page events (#776) * Add pathname to prev page events * Add a test for pathname * Change window import --- src/__tests__/page-view.ts | 9 +++++++ src/page-view.ts | 48 +++++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/__tests__/page-view.ts b/src/__tests__/page-view.ts index 14d8ac798..fbd7bf3b7 100644 --- a/src/__tests__/page-view.ts +++ b/src/__tests__/page-view.ts @@ -99,5 +99,14 @@ describe('PageView ID manager', () => { const secondPageView = pageViewIdManager.doPageView() expect(secondPageView.$prev_pageview_last_scroll).toBeDefined() }) + + it('should include the pathname', () => { + const pageViewIdManager = new PageViewManager() + + const firstPageView = pageViewIdManager.doPageView() + expect(firstPageView.$prev_pageview_pathname).toBeUndefined() + const secondPageView = pageViewIdManager.doPageView() + expect(secondPageView.$prev_pageview_pathname).toEqual('/pathname') + }) }) }) diff --git a/src/page-view.ts b/src/page-view.ts index 665e122ab..d2beb9248 100644 --- a/src/page-view.ts +++ b/src/page-view.ts @@ -1,6 +1,7 @@ -import * as utils from './utils' +import { window } from './utils' interface PageViewData { + pathname: string // scroll is how far down the page the user has scrolled, // content is how far down the page the user can view content // (e.g. if the page is 1000 tall, but the user's screen is only 500 tall, @@ -24,15 +25,21 @@ interface ScrollProperties { $prev_pageview_max_content_percentage?: number } +interface PageViewEventProperties extends ScrollProperties { + $prev_pageview_pathname?: string +} + export class PageViewManager { _pageViewData: PageViewData | undefined _hasSeenPageView = false _createPageViewData(): PageViewData { - return {} + return { + pathname: window.location.pathname, + } } - doPageView(): ScrollProperties { + doPageView(): PageViewEventProperties { let prevPageViewData: PageViewData | undefined // if there were events created before the first PageView, we would have created a // pageViewData for them. If this happened, we don't want to create a new pageViewData @@ -51,12 +58,18 @@ export class PageViewManager { // of the event loop setTimeout(this._updateScrollData, 0) - return this._calculatePrevPageScrollProperties(prevPageViewData) + return { + $prev_pageview_pathname: prevPageViewData?.pathname, + ...this._calculatePrevPageScrollProperties(prevPageViewData), + } } - doPageLeave(): ScrollProperties { + doPageLeave(): PageViewEventProperties { const prevPageViewData = this._pageViewData - return this._calculatePrevPageScrollProperties(prevPageViewData) + return { + $prev_pageview_pathname: prevPageViewData?.pathname, + ...this._calculatePrevPageScrollProperties(prevPageViewData), + } } _calculatePrevPageScrollProperties(prevPageViewData: PageViewData | undefined): ScrollProperties { @@ -121,34 +134,31 @@ export class PageViewManager { } startMeasuringScrollPosition() { - utils.window.addEventListener('scroll', this._updateScrollData) - utils.window.addEventListener('scrollend', this._updateScrollData) - utils.window.addEventListener('resize', this._updateScrollData) + window.addEventListener('scroll', this._updateScrollData) + window.addEventListener('scrollend', this._updateScrollData) + window.addEventListener('resize', this._updateScrollData) } stopMeasuringScrollPosition() { - utils.window.removeEventListener('scroll', this._updateScrollData) - utils.window.removeEventListener('scrollend', this._updateScrollData) - utils.window.removeEventListener('resize', this._updateScrollData) + window.removeEventListener('scroll', this._updateScrollData) + window.removeEventListener('scrollend', this._updateScrollData) + window.removeEventListener('resize', this._updateScrollData) } _scrollHeight(): number { - return Math.max( - 0, - utils.window.document.documentElement.scrollHeight - utils.window.document.documentElement.clientHeight - ) + return Math.max(0, window.document.documentElement.scrollHeight - window.document.documentElement.clientHeight) } _scrollY(): number { - return utils.window.scrollY || utils.window.pageYOffset || utils.window.document.documentElement.scrollTop || 0 + return window.scrollY || window.pageYOffset || window.document.documentElement.scrollTop || 0 } _contentHeight(): number { - return utils.window.document.documentElement.scrollHeight || 0 + return window.document.documentElement.scrollHeight || 0 } _contentY(): number { - const clientHeight = utils.window.document.documentElement.clientHeight || 0 + const clientHeight = window.document.documentElement.clientHeight || 0 return this._scrollY() + clientHeight } } From 9abda6d7d6cafcdf4f4f1b394def46f3c52bd27a Mon Sep 17 00:00:00 2001 From: robbie-c Date: Tue, 22 Aug 2023 06:26:05 +0000 Subject: [PATCH 09/13] chore: Bump version to 1.77.1 --- CHANGELOG.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b15f1f29..8fe462254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.77.1 - 2023-08-22 + +- feat: Add pathname to prev page events (#776) +- fix: Mitigate testcafe flakiness (#779) +- feat: Filter out events from GPTBot (#772) + ## 1.77.0 - 2023-08-18 - feat: Add previous page properties to page events (#773) diff --git a/package.json b/package.json index db36da4a6..ed217ad10 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-js", - "version": "1.77.0", + "version": "1.77.1", "description": "Posthog-js allows you to automatically capture usage and send events to PostHog.", "repository": "https://github.com/PostHog/posthog-js", "author": "hey@posthog.com", From 74a7d3761fc1398b0195cb0450e0d19c01ed5152 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 25 Aug 2023 17:49:45 +0200 Subject: [PATCH 10/13] fix(autocapture): element properties tracked up to 1k bytes (#783) --- src/__tests__/autocapture.js | 24 ++++++++++++++++++++++++ src/autocapture.ts | 13 ++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/__tests__/autocapture.js b/src/__tests__/autocapture.js index 0fc7dbdbf..2dff85193 100644 --- a/src/__tests__/autocapture.js +++ b/src/__tests__/autocapture.js @@ -708,6 +708,30 @@ describe('Autocapture system', () => { expect(props['$elements'][props['$elements'].length - 1]).toHaveProperty('tag_name', 'body') }) + it('truncate any element property value to 1024 bytes', () => { + const elTarget = document.createElement('a') + elTarget.setAttribute('href', 'http://test.com') + const longString = 'prop'.repeat(400) + elTarget.dataset.props = longString + const elParent = document.createElement('span') + elParent.appendChild(elTarget) + const elGrandparent = document.createElement('div') + elGrandparent.appendChild(elParent) + const elGreatGrandparent = document.createElement('table') + elGreatGrandparent.appendChild(elGrandparent) + document.body.appendChild(elGreatGrandparent) + const e = { + target: elTarget, + type: 'click', + } + autocapture._captureEvent(e, lib) + expect(lib.capture.calledOnce).toBe(true) + const captureArgs = lib.capture.args[0] + const props = captureArgs[1] + expect(longString).toBe('prop'.repeat(400)) + expect(props['$elements'][0]).toHaveProperty('attr__data-props', 'prop'.repeat(256) + '...') + }) + it('gets the href attribute from parent anchor tags', () => { const elTarget = document.createElement('img') const elParent = document.createElement('span') diff --git a/src/autocapture.ts b/src/autocapture.ts index 9b5072162..85cc13f7f 100644 --- a/src/autocapture.ts +++ b/src/autocapture.ts @@ -29,6 +29,13 @@ import { AutocaptureConfig, AutoCaptureCustomProperty, DecideResponse, Propertie import { PostHog } from './posthog-core' import { AUTOCAPTURE_DISABLED_SERVER_SIDE } from './constants' +function limitText(length: number, text: string): string { + if (text.length > length) { + return text.slice(0, length) + '...' + } + return text +} + const autocapture = { _initializedTokens: [] as string[], _isDisabledServerSide: null as boolean | null, @@ -82,9 +89,9 @@ const autocapture = { } if (autocaptureCompatibleElements.indexOf(tag_name) > -1 && !maskText) { if (tag_name.toLowerCase() === 'a' || tag_name.toLowerCase() === 'button') { - props['$el_text'] = getDirectAndNestedSpanText(elem) + props['$el_text'] = limitText(1024, getDirectAndNestedSpanText(elem)) } else { - props['$el_text'] = getSafeText(elem) + props['$el_text'] = limitText(1024, getSafeText(elem)) } } @@ -99,7 +106,7 @@ const autocapture = { if (isSensitiveElement(elem) && ['name', 'id', 'class'].indexOf(attr.name) === -1) return if (!maskInputs && shouldCaptureValue(attr.value) && !isAngularStyleAttr(attr.name)) { - props['attr__' + attr.name] = attr.value + props['attr__' + attr.name] = limitText(1024, attr.value) } }) From 79bfc90b993bbc683c1f251b8834130dcd420af1 Mon Sep 17 00:00:00 2001 From: timgl Date: Fri, 25 Aug 2023 15:51:08 +0000 Subject: [PATCH 11/13] chore: Bump version to 1.77.2 --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe462254..d6486625f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.77.2 - 2023-08-25 + +- fix(autocapture): element properties tracked up to 1k bytes (#783) + ## 1.77.1 - 2023-08-22 - feat: Add pathname to prev page events (#776) diff --git a/package.json b/package.json index ed217ad10..1f57ad1e4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-js", - "version": "1.77.1", + "version": "1.77.2", "description": "Posthog-js allows you to automatically capture usage and send events to PostHog.", "repository": "https://github.com/PostHog/posthog-js", "author": "hey@posthog.com", From 92edb6784ec53394c5ebd6595261d6e6ded4cf46 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 5 Sep 2023 12:36:52 +0100 Subject: [PATCH 12/13] =?UTF-8?q?feat:=20test=20a=20better=20list=20of=20b?= =?UTF-8?q?ots=20and=20allow=20users=20to=20configure=20the=20bot=E2=80=A6?= =?UTF-8?q?=20(#788)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: test a better list of bots and allow users to configure the bot list * fix * Fix * fix --- src/__tests__/utils.js | 19 ++++++++++++- src/posthog-core.ts | 3 ++- src/types.ts | 4 +++ src/utils.ts | 60 +++++++++++++++++++++++++++++++++++------- 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/__tests__/utils.js b/src/__tests__/utils.js index a3ca73dfd..b60b98b70 100644 --- a/src/__tests__/utils.js +++ b/src/__tests__/utils.js @@ -5,7 +5,13 @@ * currently not supported in the browser lib). */ -import { _copyAndTruncateStrings, _info, loadScript } from '../utils' +import { _copyAndTruncateStrings, _info, _isBlockedUA, DEFAULT_BLOCKED_UA_STRS, loadScript } from '../utils' + +function userAgentFor(botString) { + const randOne = (Math.random() + 1).toString(36).substring(7) + const randTwo = (Math.random() + 1).toString(36).substring(7) + return `Mozilla/5.0 (compatible; ${botString}/${randOne}; +http://a.com/bot/${randTwo})` +} describe(`utils.js`, () => { it('should have $host and $pathname in properties', () => { @@ -208,4 +214,15 @@ describe('loadScript', () => { new_script.onerror('uh-oh') expect(callback).toHaveBeenCalledWith('uh-oh') }) + + describe('user agent blocking', () => { + it.each(DEFAULT_BLOCKED_UA_STRS.concat('testington'))( + 'blocks a bot based on the user agent %s', + (botString) => { + const randomisedUserAgent = userAgentFor(botString) + + expect(_isBlockedUA(randomisedUserAgent, ['testington'])).toBe(true) + } + ) + }) }) diff --git a/src/posthog-core.ts b/src/posthog-core.ts index 81826d2f3..edd056947 100644 --- a/src/posthog-core.ts +++ b/src/posthog-core.ts @@ -114,6 +114,7 @@ const defaultConfig = (): PostHogConfig => ({ loaded: __NOOP, store_google: true, custom_campaign_params: [], + custom_blocked_useragents: [], save_referrer: true, test: false, verbose: false, @@ -853,7 +854,7 @@ export class PostHog { return } - if (_isBlockedUA(userAgent)) { + if (_isBlockedUA(userAgent, this.get_config('custom_blocked_useragents'))) { return } diff --git a/src/types.ts b/src/types.ts index e39a4f2a1..a041dc894 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,6 +63,10 @@ export interface PostHogConfig { loaded: (posthog_instance: PostHog) => void store_google: boolean custom_campaign_params: string[] + // a list of strings to be tested against navigator.userAgent to determine if the source is a bot + // this is **added to** the default list of bots that we check + // defaults to the empty array + custom_blocked_useragents: string[] save_referrer: boolean test: boolean verbose: boolean diff --git a/src/utils.ts b/src/utils.ts index 0a786882c..2ee2ab003 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -461,18 +461,60 @@ export const _utf8Encode = function (string: string): string { return utftext } +export const DEFAULT_BLOCKED_UA_STRS = [ + 'ahrefsbot', + 'applebot', + 'baiduspider', + 'bingbot', + 'bingpreview', + 'bot.htm', + 'bot.php', + 'crawler', + 'duckduckbot', + 'facebookexternal', + 'facebookcatalog', + 'gptbot', + 'hubspot', + 'linkedinbot', + 'mj12bot', + 'petalbot', + 'pinterest', + 'prerender', + 'rogerbot', + 'screaming frog', + 'semrushbot', + 'sitebulb', + 'twitterbot', + 'yahoo! slurp', + 'yandexbot', + + // a whole bunch of goog-specific crawlers + // https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers + 'adsbot-google', + 'apis-google', + 'duplexweb-google', + 'feedfetcher-google', + 'google favicon', + 'google web preview', + 'google-read-aloud', + 'googlebot', + 'googleweblight', + 'mediapartners-google', + 'storebot-google', +] + // _.isBlockedUA() // This is to block various web spiders from executing our JS and // sending false capturing data -export const _isBlockedUA = function (ua: string): boolean { - if ( - /(google web preview|baiduspider|yandexbot|bingbot|googlebot|yahoo! slurp|ahrefsbot|facebookexternalhit|facebookcatalog|applebot|semrushbot|duckduckbot|twitterbot|rogerbot|linkedinbot|mj12bot|sitebulb|bot.htm|bot.php|hubspot|crawler|prerender|gptbot)/i.test( - ua - ) - ) { - return true - } - return false +export const _isBlockedUA = function (ua: string, customBlockedUserAgents: string[]): boolean { + return DEFAULT_BLOCKED_UA_STRS.concat(customBlockedUserAgents).some((blockedUA) => { + if (ua.includes) { + return ua.includes(blockedUA) + } else { + // IE 11 :/ + return ua.indexOf(blockedUA) !== -1 + } + }) } /** From 35e599260e0120116c885ef6c0356a215282e2dd Mon Sep 17 00:00:00 2001 From: pauldambra Date: Tue, 5 Sep 2023 11:38:22 +0000 Subject: [PATCH 13/13] chore: Bump version to 1.77.3 --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6486625f..c904b401f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.77.3 - 2023-09-05 + +- feat: test a better list of bots and allow users to configure the bot… (#788) + ## 1.77.2 - 2023-08-25 - fix(autocapture): element properties tracked up to 1k bytes (#783) diff --git a/package.json b/package.json index 1f57ad1e4..5870e04f8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-js", - "version": "1.77.2", + "version": "1.77.3", "description": "Posthog-js allows you to automatically capture usage and send events to PostHog.", "repository": "https://github.com/PostHog/posthog-js", "author": "hey@posthog.com",