From bd43064cbd68e79752b26f32e0423592bb8656f7 Mon Sep 17 00:00:00 2001 From: Steen3S <3s@steenberghe.com> Date: Thu, 15 Aug 2024 15:56:09 +0200 Subject: [PATCH] Tracking pageviews with middleware --- app/api/track/route.ts | 13 +++++ lib/databases/analytics.ts | 24 +++++++++ lib/databases/clickhouse.ts | 12 +++++ lib/databases/table.md | 15 ++++++ middleware.ts | 102 +++++++++++++++++++++++++++++++++++- package.json | 4 +- yarn.lock | 58 ++++++++++++++++++++ 7 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 app/api/track/route.ts create mode 100644 lib/databases/analytics.ts create mode 100644 lib/databases/clickhouse.ts create mode 100644 lib/databases/table.md diff --git a/app/api/track/route.ts b/app/api/track/route.ts new file mode 100644 index 000000000..18337ce16 --- /dev/null +++ b/app/api/track/route.ts @@ -0,0 +1,13 @@ +import { PageViewSchema, trackPageView } from 'lib/databases/analytics'; + +export const POST = async (req: Request) => { + const _body = await req.json(); + + const data = await PageViewSchema.validate(_body, { stripUnknown: true }).catch(() => {}); + + if (data) { + await trackPageView(data).catch(() => {}); + } + + return new Response('OK'); +}; diff --git a/lib/databases/analytics.ts b/lib/databases/analytics.ts new file mode 100644 index 000000000..cd612c2e7 --- /dev/null +++ b/lib/databases/analytics.ts @@ -0,0 +1,24 @@ +import { InferType, object, string } from 'yup'; +import { getClickHouse } from './clickhouse'; + +const client = getClickHouse(); + +export const PageViewSchema = object({ + path: string().required(), + affiliate: string(), + referrer: string(), + agent: string(), + hostname: string().required(), +}); + +export type PageViewData = InferType; + +const tableName = 'page_view'; + +export const trackPageView = async (data: PageViewData) => { + await client.insert({ + table: tableName, + values: [data], + format: 'JSONEachRow', + }); +}; diff --git a/lib/databases/clickhouse.ts b/lib/databases/clickhouse.ts new file mode 100644 index 000000000..c259336e1 --- /dev/null +++ b/lib/databases/clickhouse.ts @@ -0,0 +1,12 @@ +import { createClient } from '@clickhouse/client'; + +let client: undefined | ReturnType; + +export const getClickHouse = () => { + if (!client) { + client = createClient({ + url: process.env.CLICKHOUSE_URL, + }); + } + return client; +}; diff --git a/lib/databases/table.md b/lib/databases/table.md new file mode 100644 index 000000000..2f6031eba --- /dev/null +++ b/lib/databases/table.md @@ -0,0 +1,15 @@ +Clickhousedb table for tracking page views + +```sql +CREATE TABLE page_view ( + event_date Date DEFAULT today(), -- Stores the date of the event, defaulting to the current date + event_date_time DateTime DEFAULT now(), -- Timestamp of the event + path String, -- Path of the page + affiliate String, -- Affiliate identifier + referrer String, -- URL of the referrer + agent String, -- User agent of the client + hostname String -- Hostname where the event was recorded +) ENGINE = MergeTree() +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event, customer, source); -- Adjust the ORDER BY clause based on your query patterns +``` diff --git a/middleware.ts b/middleware.ts index ba75938f6..0914d7566 100644 --- a/middleware.ts +++ b/middleware.ts @@ -1,7 +1,107 @@ +import { PageViewData } from 'lib/databases/analytics'; import { defaultLocale, localePrefix, locales } from 'lib/i18n/config'; import createMiddleware from 'next-intl/middleware'; +import { NextRequest } from 'next/server'; -export default createMiddleware({ locales, localePrefix, defaultLocale }); +export const middleware = (request: NextRequest) => { + const isPageView = isValidPageView(request); + + if (isPageView) { + trackPageView(request); + } + + return createMiddleware({ locales, localePrefix, defaultLocale })(request); +}; + +const trackPageView = async (request: NextRequest) => { + const server = getURL('/api/track'); + + const url = new URL(request.url); + const path = url.pathname; + const referrer = request.headers.get('referer'); + const agent = request.headers.get('user-agent'); + + const data: PageViewData = { + path, + ...(referrer && { referrer }), + ...(referrer && { referrer }), + ...(agent && { agent }), + hostname: server.hostname, + }; + + // Send data to /api/track + fetch(server.href, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + name: 'page-view', + ...data, + }), + }).catch(() => {}); +}; + +/** + * Get the URL of the current (vercel) deployment + */ +const getURL = (path?: `/${string}`) => { + const hostname = + process.env.NEXT_PUBLIC_VERCEL_PROJECT_PRODUCTION_URL || + process.env.VERCEL_PROJECT_PRODUCTION_URL || + process.env.VERCEL_URL || + 'localhost:3000'; + const isDev = process.env.NODE_ENV !== 'production'; + + const protocol = isDev ? 'http' : 'https'; + + return new URL(`${protocol}://${hostname}${path || ''}`); +}; + +const isValidPageView = (request: NextRequest) => { + // List of regex patterns to match against the user-agent header + const botUserAgents = [ + '.*bot.*', + '.*preview.*', + '.*crawler.*', + '.*google.*', + '.*bing.*', + '.*yahoo.*', + '.*baidu.*', + '.*yandex.*', + '.*duckduckgo.*', + '.*facebook.*', + '.*twitter.*', + '.*instagram.*', + '.*pinterest.*', + '.*linkedin.*', + ]; + + // Check if the user-agent header matches any of the bot patterns + const isBot = botUserAgents.some((pattern) => { + const regex = new RegExp(pattern, 'i'); + return regex.test(request.headers.get('user-agent')); + }); + if (isBot) return false; + + const ref = new URL(request.headers.get('referer')); + const url = new URL(request.url); + + const isNavigating = request.headers.has('next-url'); + if (isNavigating) return false; + + // Check if the request is a navigation + const isNavigation = ref.pathname !== url.pathname; + + if (isNavigation) return true; + + // Check if the request is a page view + const isPageView = request.headers.get('sec-fetch-mode') === 'navigate'; + + if (!isPageView) return false; + + return true; +}; export const config = { // Allow all paths starting with /address and apply exclusions to other paths diff --git a/package.json b/package.json index f8f4a2a0b..c2275eb9c 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ }, "private": true, "dependencies": { + "@clickhouse/client": "^1.4.1", "@headlessui/react": "^1.7.19", "@heroicons/react": "^2.1.3", "@revoke.cash/chains": "^47.0.0", @@ -64,7 +65,8 @@ "timeago.js": "^4.0.2", "use-local-storage": "^3.0.0", "viem": "^2.9.5", - "wagmi": "^2.5.13" + "wagmi": "^2.5.13", + "yup": "^1.4.0" }, "devDependencies": { "@cypress/grep": "^4.0.1", diff --git a/yarn.lock b/yarn.lock index 7929092ca..33394fddb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -143,6 +143,22 @@ __metadata: languageName: node linkType: hard +"@clickhouse/client-common@npm:1.4.1": + version: 1.4.1 + resolution: "@clickhouse/client-common@npm:1.4.1" + checksum: 69bd0c9e3d4ab2135f73fa4a8e60bc5c1ee7ca4e7cd036e83b849d3a36166401c1d6da114da43e31c63addb5f9dabfe157b7d77a236644275a6b319a9c9d1e3b + languageName: node + linkType: hard + +"@clickhouse/client@npm:^1.4.1": + version: 1.4.1 + resolution: "@clickhouse/client@npm:1.4.1" + dependencies: + "@clickhouse/client-common": 1.4.1 + checksum: 462e37289f5d57fbb7d49aa1f1d1e4b2f05b76a8a19080e5bcd9b16b596472cc542fcd61da6eeab05b02f38b677e5d2e8a01564163a589b6a0351714323ab382 + languageName: node + linkType: hard + "@coinbase/wallet-sdk@npm:3.9.1": version: 3.9.1 resolution: "@coinbase/wallet-sdk@npm:3.9.1" @@ -9852,6 +9868,13 @@ __metadata: languageName: node linkType: hard +"property-expr@npm:^2.0.5": + version: 2.0.6 + resolution: "property-expr@npm:2.0.6" + checksum: 89977f4bb230736c1876f460dd7ca9328034502fd92e738deb40516d16564b850c0bbc4e052c3df88b5b8cd58e51c93b46a94bea049a3f23f4a022c038864cab + languageName: node + linkType: hard + "property-information@npm:^5.0.0": version: 5.6.0 resolution: "property-information@npm:5.6.0" @@ -10626,6 +10649,7 @@ __metadata: version: 0.0.0-use.local resolution: "revoke.cash@workspace:." dependencies: + "@clickhouse/client": ^1.4.1 "@cypress/grep": ^4.0.1 "@headlessui/react": ^1.7.19 "@heroicons/react": ^2.1.3 @@ -10701,6 +10725,7 @@ __metadata: viem: ^2.9.5 wagmi: ^2.5.13 walkdir: ^0.4.1 + yup: ^1.4.0 languageName: unknown linkType: soft @@ -11761,6 +11786,13 @@ __metadata: languageName: node linkType: hard +"tiny-case@npm:^1.0.3": + version: 1.0.3 + resolution: "tiny-case@npm:1.0.3" + checksum: 3f7a30c39d5b0e1bc097b0b271bec14eb5b836093db034f35a0de26c14422380b50dc12bfd37498cf35b192f5df06f28a710712c87ead68872a9e37ad6f6049d + languageName: node + linkType: hard + "tippy.js@npm:^6.3.1": version: 6.3.7 resolution: "tippy.js@npm:6.3.7" @@ -11802,6 +11834,13 @@ __metadata: languageName: node linkType: hard +"toposort@npm:^2.0.2": + version: 2.0.2 + resolution: "toposort@npm:2.0.2" + checksum: d64c74b570391c9432873f48e231b439ee56bc49f7cb9780b505cfdf5cb832f808d0bae072515d93834dd6bceca5bb34448b5b4b408335e4d4716eaf68195dcb + languageName: node + linkType: hard + "totalist@npm:^3.0.0": version: 3.0.1 resolution: "totalist@npm:3.0.1" @@ -11923,6 +11962,13 @@ __metadata: languageName: node linkType: hard +"type-fest@npm:^2.19.0": + version: 2.19.0 + resolution: "type-fest@npm:2.19.0" + checksum: a4ef07ece297c9fba78fc1bd6d85dff4472fe043ede98bd4710d2615d15776902b595abf62bd78339ed6278f021235fb28a96361f8be86ed754f778973a0d278 + languageName: node + linkType: hard + "type-fest@npm:^3.0.0": version: 3.13.1 resolution: "type-fest@npm:3.13.1" @@ -12998,6 +13044,18 @@ __metadata: languageName: node linkType: hard +"yup@npm:^1.4.0": + version: 1.4.0 + resolution: "yup@npm:1.4.0" + dependencies: + property-expr: ^2.0.5 + tiny-case: ^1.0.3 + toposort: ^2.0.2 + type-fest: ^2.19.0 + checksum: 20a2ee0c1e891979ca16b34805b3a3be9ab4bea6ea3d2f9005b998b4dc992d0e4d7b53e5f4d8d9423420046630fb44fdf0ecf7e83bc34dd83392bca046c5229d + languageName: node + linkType: hard + "zustand@npm:4.4.1": version: 4.4.1 resolution: "zustand@npm:4.4.1"