From 19e4e637ede50a33e95dc8793f2aefb9931fbab4 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 19 Oct 2023 11:01:52 +0800 Subject: [PATCH 01/15] feat(ga4): add ga4 lib --- bin/sync-ga4-data.ts | 15 +++ lib/ga4.ts | 173 ++++++++++++++++++++++++++++++ package-lock.json | 250 ++++++++++++++++++++++++++++++++++++------- package.json | 1 + 4 files changed, 401 insertions(+), 38 deletions(-) create mode 100644 bin/sync-ga4-data.ts create mode 100644 lib/ga4.ts diff --git a/bin/sync-ga4-data.ts b/bin/sync-ga4-data.ts new file mode 100644 index 0000000..c44d890 --- /dev/null +++ b/bin/sync-ga4-data.ts @@ -0,0 +1,15 @@ +import fs from "fs"; + +import { convertAndMerge, saveGA4Data } from "../lib/ga4.js"; + +// const data = JSON.parse(fs.readFileSync("data.json", "utf8")); +// convertAndMerge(data).then(console.log); + +saveGA4Data( + { "3": 1, "2": 4 }, + { startDate: "2021-10-29", endDate: "2021-10-29" } +).then(console.log); + +//fetchData({startDate: '2021-10-29', endDate: '2021-10-29'}).then(res => { +// fs.writeFileSync('data.json', JSON.stringify(res, null, 2)) +//}); diff --git a/lib/ga4.ts b/lib/ga4.ts new file mode 100644 index 0000000..100ce95 --- /dev/null +++ b/lib/ga4.ts @@ -0,0 +1,173 @@ +import { BetaAnalyticsDataClient } from "@google-analytics/data"; + +import { pgKnexRO as knexRO, pgKnex as knex } from "./db.js"; + +const propertyId = process.env.MATTERS_GA4_PROPERTY_ID; + +const analyticsDataClient = new BetaAnalyticsDataClient(); + +interface Row { + path: string; + totalUsers: string; +} + +interface MergedData { + [key: string]: number; +} + +export const fetchGA4Data = async ({ + startDate, + endDate, +}: { + startDate: string; + endDate: string; +}): Promise => { + const limit = 10000; + let offset = 0; + const result: Row[] = []; + while (true) { + const res = await request({ startDate, endDate, limit, offset }); + result.push(...res); + offset += limit; + if (res.length < limit) { + break; + } + } + return result; +}; + +export const saveGA4Data = async ( + data: MergedData, + { startDate, endDate }: { startDate: string; endDate: string } +) => { + const rows = Object.entries(data).map(([id, totalUsers]) => ({ + articleId: id, + totalUsers, + dateRange: `[${startDate}, ${endDate}]`, + })); + const table = "article_ga4_data"; + const updateRows = [] + const insertRows = [] + for (const { articleId, dateRange, totalUsers } of rows) { + const res = await knexRO(table) + .where({ articleId, dateRange }) + .select("id", "totalUsers") + .first(); + if (res && res.totalUsers) { + if (res.totalUsers !== String(totalUsers)) { + // only update when totalUsers is different + updateRows.push({id: res.id, totalUsers}) + } + } else { + insertRows.push({ articleId, dateRange, totalUsers }); + } + } + if (updateRows.length > 0) { + for (const {id, totalUsers} of updateRows) { + await knex(table).update({totalUsers}).where({id: id}) + } + } + if (insertRows.length > 0) { + await knex(table).insert(insertRows); + } +}; + +export const convertAndMerge = async (rows: Row[]): Promise => { + const converted = Promise.all( + rows.map(async (row) => ({ + id: await pathToId(row.path), + totalUsers: parseInt(row.totalUsers, 10), + })) + ); + const res: MergedData = {}; + for (const row of await converted) { + if (row.id in res) { + res[row.id] += row.totalUsers; + } else { + res[row.id] = row.totalUsers; + } + } + return res; +}; + +const pathToId = async (path: string) => { + const [_, __, articlePath] = path.split("/"); + if (articlePath) { + const parts = articlePath.split("-"); + const idLike = parts[0]; + const hash = parts[parts.length - 1]; + if (!isNaN(parseInt(idLike))) { + return idLike; + } else { + return hashToId(hash); + } + } +}; + +const hashToId = async (hash: string) => { + const res = await knexRO("article") + .where({ mediaHash: hash }) + .select("id") + .first(); + if (res) { + return res.id; + } else { + return null; + } +}; + +// https://developers.google.com/analytics/devguides/reporting/data/v1 +const request = async ({ + startDate, + endDate, + limit, + offset, +}: { + startDate: string; + endDate: string; + limit: number; + offset: number; +}): Promise => { + const [response] = await analyticsDataClient.runReport({ + property: `properties/${propertyId}`, + dateRanges: [ + { + startDate, + endDate, + }, + ], + dimensions: [ + { + name: "pagePath", + }, + ], + dimensionFilter: { + filter: { + fieldName: "pagePath", + stringFilter: { + matchType: "BEGINS_WITH", + value: "/@", + }, + }, + }, + metrics: [ + { + name: "totalUsers", + //name: 'activeUsers', + }, + ], + limit, + offset, + returnPropertyQuota: true, + }); + if (response && response.rows) { + console.log(response.propertyQuota); + console.log(`total rows count: ${response.rowCount}`); + return response.rows.map((row) => ({ + path: (row.dimensionValues && row.dimensionValues[0].value) ?? "", + totalUsers: (row.metricValues && row.metricValues[0].value) ?? "0", + })); + } else { + throw new Error("No response received."); + } +}; diff --git a/package-lock.json b/package-lock.json index c81d682..e245b51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.267.0", "@aws-sdk/client-sqs": "^3.266.0", + "@google-analytics/data": "^4.0.1", "@google-cloud/bigquery": "^7.3.0", "@matters/apollo-response-cache": "^1.4.0-rc.0", "@matters/ipns-site-generator": "^0.1.3", @@ -3691,6 +3692,17 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@google-analytics/data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-analytics/data/-/data-4.0.1.tgz", + "integrity": "sha512-YhxMQuno21LF7W9sLZMPBuxJoRQ/A50kFvGJuZUmD4girm5PyEQDpew6vooUh7GjMcfpvM8b1XIIFk6pyFaBjA==", + "dependencies": { + "google-gax": "^4.0.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@google-cloud/bigquery": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.3.0.tgz", @@ -3781,6 +3793,40 @@ "graphql": "^14.0.0 || ^15.0.0" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.5.tgz", + "integrity": "sha512-iouYNlPxRAwZ2XboDT+OfRKHuaKHiqjB5VFYZ0NFrHkbEF+AV3muIUY9olQsp8uxU4VvRCMiRk9ftzFDGb61aw==", + "dependencies": { + "@grpc/proto-loader": "^0.7.8", + "@types/node": ">=12.12.47" + }, + "engines": { + "node": "^8.13.0 || >=10.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.10", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.10.tgz", + "integrity": "sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.4", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -5282,6 +5328,17 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", @@ -5378,7 +5435,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -6231,7 +6287,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -6277,7 +6332,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -6288,8 +6342,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/color-support": { "version": "1.1.3", @@ -7179,6 +7232,14 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", @@ -7583,7 +7644,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -7743,6 +7803,27 @@ "node": ">=14" } }, + "node_modules/google-gax": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.0.4.tgz", + "integrity": "sha512-Yoey/ABON2HaTUIRUt5tTQAvwQ6E/2etSyFXwHNVcYtIiYDpKix7G4oorZdkp17gFiYovzRCRhRZYrfdCgRK9Q==", + "dependencies": { + "@grpc/grpc-js": "~1.9.0", + "@grpc/proto-loader": "^0.7.0", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "google-auth-library": "^9.0.0", + "node-fetch": "^2.6.1", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^2.0.0", + "protobufjs": "7.2.5", + "retry-request": "^6.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -9549,6 +9630,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "node_modules/lodash.deburr": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", @@ -10321,6 +10407,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", @@ -11008,10 +11102,21 @@ "node": ">= 6" } }, + "node_modules/proto3-json-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.0.tgz", + "integrity": "sha512-FB/YaNrpiPkyQNSNPilpn8qn0KdEfkgmJ9JP93PQyF/U4bAiXY5BiUdDhiDO4S48uSQ6AesklgVlrKiqZPzegw==", + "dependencies": { + "protobufjs": "^7.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/protobufjs": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.0.tgz", - "integrity": "sha512-hYCqTDuII4iJ4stZqiuGCSU8xxWl5JeXYpwARGtn/tWcKCAro6h3WQz+xpsNbXW0UYqpmTQFEyFWO0G0Kjt64g==", + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", + "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", "hasInstallScript": true, "dependencies": { "@protobufjs/aspromise": "^1.1.2", @@ -11258,7 +11363,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -12327,7 +12431,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -12370,7 +12473,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -12381,10 +12483,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -12402,7 +12503,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } @@ -15626,6 +15726,14 @@ "strip-json-comments": "^3.1.1" } }, + "@google-analytics/data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-analytics/data/-/data-4.0.1.tgz", + "integrity": "sha512-YhxMQuno21LF7W9sLZMPBuxJoRQ/A50kFvGJuZUmD4girm5PyEQDpew6vooUh7GjMcfpvM8b1XIIFk6pyFaBjA==", + "requires": { + "google-gax": "^4.0.3" + } + }, "@google-cloud/bigquery": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.3.0.tgz", @@ -15695,6 +15803,33 @@ "tslib": "~2.2.0" } }, + "@grpc/grpc-js": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.5.tgz", + "integrity": "sha512-iouYNlPxRAwZ2XboDT+OfRKHuaKHiqjB5VFYZ0NFrHkbEF+AV3muIUY9olQsp8uxU4VvRCMiRk9ftzFDGb61aw==", + "requires": { + "@grpc/proto-loader": "^0.7.8", + "@types/node": ">=12.12.47" + } + }, + "@grpc/proto-loader": { + "version": "0.7.10", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.10.tgz", + "integrity": "sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==", + "requires": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.4", + "yargs": "^17.7.2" + }, + "dependencies": { + "long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + } + } + }, "@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -16889,6 +17024,14 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "acorn": { "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", @@ -16954,7 +17097,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -17578,7 +17720,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -17611,7 +17752,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -17619,8 +17759,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "color-support": { "version": "1.1.3", @@ -18316,6 +18455,11 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", @@ -18628,8 +18772,7 @@ "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-intrinsic": { "version": "1.2.0", @@ -18738,6 +18881,24 @@ "lru-cache": "^6.0.0" } }, + "google-gax": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.0.4.tgz", + "integrity": "sha512-Yoey/ABON2HaTUIRUt5tTQAvwQ6E/2etSyFXwHNVcYtIiYDpKix7G4oorZdkp17gFiYovzRCRhRZYrfdCgRK9Q==", + "requires": { + "@grpc/grpc-js": "~1.9.0", + "@grpc/proto-loader": "^0.7.0", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "google-auth-library": "^9.0.0", + "node-fetch": "^2.6.1", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^2.0.0", + "protobufjs": "7.2.5", + "retry-request": "^6.0.0" + } + }, "gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -20060,6 +20221,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "lodash.deburr": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", @@ -20693,6 +20859,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" + }, "object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", @@ -21192,10 +21363,18 @@ "sisteransi": "^1.0.5" } }, + "proto3-json-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.0.tgz", + "integrity": "sha512-FB/YaNrpiPkyQNSNPilpn8qn0KdEfkgmJ9JP93PQyF/U4bAiXY5BiUdDhiDO4S48uSQ6AesklgVlrKiqZPzegw==", + "requires": { + "protobufjs": "^7.0.0" + } + }, "protobufjs": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.0.tgz", - "integrity": "sha512-hYCqTDuII4iJ4stZqiuGCSU8xxWl5JeXYpwARGtn/tWcKCAro6h3WQz+xpsNbXW0UYqpmTQFEyFWO0G0Kjt64g==", + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", + "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", "requires": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", @@ -21389,8 +21568,7 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, "require-from-string": { "version": "2.0.2", @@ -22162,7 +22340,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -22192,8 +22369,7 @@ "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { "version": "4.0.0", @@ -22201,10 +22377,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "requires": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -22218,8 +22393,7 @@ "yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" }, "yn": { "version": "3.1.1", diff --git a/package.json b/package.json index 7665231..84515af 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.267.0", "@aws-sdk/client-sqs": "^3.266.0", + "@google-analytics/data": "^4.0.1", "@google-cloud/bigquery": "^7.3.0", "@matters/apollo-response-cache": "^1.4.0-rc.0", "@matters/ipns-site-generator": "^0.1.3", From 97eaf7eb486051db3e66ba4e1fa2253812a6fbd7 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 19 Oct 2023 12:43:43 +0800 Subject: [PATCH 02/15] feat(ga4): add handler --- handlers/sync-audit-log-to-bigquery.ts | 8 +++++-- handlers/sync-ga4-data.ts | 32 ++++++++++++++++++++++++++ lib/ga4.ts | 10 ++++---- 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 handlers/sync-ga4-data.ts diff --git a/handlers/sync-audit-log-to-bigquery.ts b/handlers/sync-audit-log-to-bigquery.ts index 463cbfb..7dbc430 100644 --- a/handlers/sync-audit-log-to-bigquery.ts +++ b/handlers/sync-audit-log-to-bigquery.ts @@ -80,8 +80,12 @@ const processAndDumpLocal = async ( action: data.action ? data.action.slice(0, 50) : undefined, entity: data.entity ? data.entity.slice(0, 50) : undefined, entity_id: data.entityId, - old_value: data.oldValue ? String(data.oldValue).slice(0, 255) : undefined, - new_value: data.newValue ? String(data.newValue).slice(0, 255) : undefined, + old_value: data.oldValue + ? String(data.oldValue).slice(0, 255) + : undefined, + new_value: data.newValue + ? String(data.newValue).slice(0, 255) + : undefined, status: data.status.slice(0, 10), request_id: requestId ? requestId.slice(0, 36) : undefined, ip: data.ip ? data.ip.slice(0, 45) : undefined, diff --git a/handlers/sync-ga4-data.ts b/handlers/sync-ga4-data.ts new file mode 100644 index 0000000..bba9411 --- /dev/null +++ b/handlers/sync-ga4-data.ts @@ -0,0 +1,32 @@ +import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js"; + +// envs + +// AWS EventBridge can configure the input event sent to Lambda, +// see https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-transform-target-input.html for info. +type Event = { + data: { + type: "today" | "yesterday"; + }; +}; + +// set timezone to UTC+8 +process.env.TZ = "Asia/Taipei"; + +const getDate = (type: Event["data"]["type"]) => { + const date = new Date(); + if (type === "yesterday") { + date.setDate(date.getDate() - 1); + } + return date; +}; + +export const handler = async (event: Event) => { + const { type } = event.data; + const date = getDate(type); + const startDate = date.toISOString().slice(0, 10); + const endDate = startDate; + const data = await fetchGA4Data({ startDate, endDate }); + const convertedData = await convertAndMerge(data); + await saveGA4Data(convertedData, { startDate, endDate }); +}; diff --git a/lib/ga4.ts b/lib/ga4.ts index 100ce95..57a6ecf 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -46,8 +46,8 @@ export const saveGA4Data = async ( dateRange: `[${startDate}, ${endDate}]`, })); const table = "article_ga4_data"; - const updateRows = [] - const insertRows = [] + const updateRows = []; + const insertRows = []; for (const { articleId, dateRange, totalUsers } of rows) { const res = await knexRO(table) .where({ articleId, dateRange }) @@ -56,15 +56,15 @@ export const saveGA4Data = async ( if (res && res.totalUsers) { if (res.totalUsers !== String(totalUsers)) { // only update when totalUsers is different - updateRows.push({id: res.id, totalUsers}) + updateRows.push({ id: res.id, totalUsers }); } } else { insertRows.push({ articleId, dateRange, totalUsers }); } } if (updateRows.length > 0) { - for (const {id, totalUsers} of updateRows) { - await knex(table).update({totalUsers}).where({id: id}) + for (const { id, totalUsers } of updateRows) { + await knex(table).update({ totalUsers }).where({ id: id }); } } if (insertRows.length > 0) { From c2e16b99ec6db4c85ab922d6d2055eecb45e3490 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:08:52 +0800 Subject: [PATCH 03/15] test: update matters-server repo and fix testing --- .github/workflows/test.yml | 3 +-- jest.config.cjs | 1 - lib/db.ts | 4 +--- matters-server | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9d2ca7..ca47d3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,5 +52,4 @@ jobs: MATTERS_NEW_FEATURE_TAG_ID: 1 MATTERS_PG_RO_CONNECTION_STRING: postgresql://postgres:postgres@localhost/test_matters-test MATTERS_CACHE_HOST: localhost - - + MATTERS_TEST_DB_SETUP: 1 diff --git a/jest.config.cjs b/jest.config.cjs index 29611b4..4d7b05e 100644 --- a/jest.config.cjs +++ b/jest.config.cjs @@ -6,7 +6,6 @@ module.exports = { testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$', testPathIgnorePatterns: ['/node_modules/', '/matters-server/'], globalSetup: '/matters-server/db/globalTestSetup.js', - globalTeardown: '/matters-server/db/globalTestTeardown.js', transform: { '\\.tsx?$': ['ts-jest', { useESM: true, diff --git a/lib/db.ts b/lib/db.ts index a0fd12a..4bf67d3 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -5,12 +5,10 @@ import { getKnexClient, getPostgresJsClient } from "./utils/db.js"; const CLOUDFLARE_IMAGE_ENDPOINT = process.env.CLOUDFLARE_IMAGE_ENDPOINT || ""; const MATTERS_AWS_S3_ENDPOINT = process.env.MATTERS_AWS_S3_ENDPOINT || ""; -const isTest = process.env.MATTERS_ENV === "test"; const dbHost = process.env.MATTERS_PG_HOST || ""; const dbUser = process.env.MATTERS_PG_USER || ""; const dbPasswd = process.env.MATTERS_PG_PASSWORD || ""; -const _dbName = process.env.MATTERS_PG_DATABASE || ""; -const dbName = isTest ? "test_" + _dbName : _dbName; +const dbName = process.env.MATTERS_PG_DATABASE || ""; const databaseURL = process.env.PG_CONNECTION_STRING || diff --git a/matters-server b/matters-server index 099fd78..36aa9b6 160000 --- a/matters-server +++ b/matters-server @@ -1 +1 @@ -Subproject commit 099fd78537ffa1c74507887c5a6b94ec23d6bfae +Subproject commit 36aa9b6fb8b1b649a861045a72d725a41892632b From 9702162ad167bf9370b19684f65eaa6cad8745fe Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:37:12 +0800 Subject: [PATCH 04/15] feat(ga4): add `getLocalDateString` helper --- handlers/sync-ga4-data.ts | 17 +++++++------- lib/__test__/ga4.test.ts | 34 +++++++++++++++++++++++++++ lib/ga4.ts | 49 ++++++++++++++++++++++++--------------- 3 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 lib/__test__/ga4.test.ts diff --git a/handlers/sync-ga4-data.ts b/handlers/sync-ga4-data.ts index bba9411..478922b 100644 --- a/handlers/sync-ga4-data.ts +++ b/handlers/sync-ga4-data.ts @@ -1,4 +1,9 @@ -import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js"; +import { + fetchGA4Data, + convertAndMerge, + saveGA4Data, + getLocalDateString, +} from "../lib/ga4.js"; // envs @@ -10,21 +15,17 @@ type Event = { }; }; -// set timezone to UTC+8 -process.env.TZ = "Asia/Taipei"; - -const getDate = (type: Event["data"]["type"]) => { +const getDate = (type: "today" | "yesterday") => { const date = new Date(); if (type === "yesterday") { date.setDate(date.getDate() - 1); } - return date; + return getLocalDateString(date); }; export const handler = async (event: Event) => { const { type } = event.data; - const date = getDate(type); - const startDate = date.toISOString().slice(0, 10); + const startDate = getDate(type); const endDate = startDate; const data = await fetchGA4Data({ startDate, endDate }); const convertedData = await convertAndMerge(data); diff --git a/lib/__test__/ga4.test.ts b/lib/__test__/ga4.test.ts new file mode 100644 index 0000000..7068e11 --- /dev/null +++ b/lib/__test__/ga4.test.ts @@ -0,0 +1,34 @@ +import { pgKnex as knex } from "../db"; +import { saveGA4Data, TABLE_NAME, getLocalDateString } from "../ga4"; + +test("saveGA4Data", async () => { + const startDate = "2021-01-01"; + const endDate = "2021-01-01"; + + // insert + await saveGA4Data({ "1": 1, "2": 2 }, { startDate, endDate }); + + const rows = await knex(TABLE_NAME) + .select("*") + .where({ dateRange: `[${startDate}, ${endDate}]` }); + expect(rows.length).toBe(2); + + // insert and update + await saveGA4Data({ "1": 2, "3": 3 }, { startDate, endDate }); + + const rows2 = await knex(TABLE_NAME) + .select("*") + .where({ dateRange: `[${startDate}, ${endDate}]` }); + expect(rows2.length).toBe(3); + for (const row of rows2) { + if (row.articleId === "1") { + expect(row.totalUsers).toBe("2"); + } + } +}); + +test("getLocalDateString", async () => { + const date = new Date("2021-01-01"); + const dateStr = getLocalDateString(date); + expect(dateStr).toBe("2021-01-01"); +}); diff --git a/lib/ga4.ts b/lib/ga4.ts index 57a6ecf..f00b101 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -4,7 +4,7 @@ import { pgKnexRO as knexRO, pgKnex as knex } from "./db.js"; const propertyId = process.env.MATTERS_GA4_PROPERTY_ID; -const analyticsDataClient = new BetaAnalyticsDataClient(); +export const TABLE_NAME = "article_ga4_data"; interface Row { path: string; @@ -15,6 +15,11 @@ interface MergedData { [key: string]: number; } +export const getLocalDateString = (date: Date) => { + // return utc+8 date string in YYYY-MM-DD format + return date.toLocaleDateString("sv", { timeZone: "Asia/Taipei" }); +}; + export const fetchGA4Data = async ({ startDate, endDate, @@ -22,11 +27,15 @@ export const fetchGA4Data = async ({ startDate: string; endDate: string; }): Promise => { + const analyticsDataClient = new BetaAnalyticsDataClient(); const limit = 10000; let offset = 0; const result: Row[] = []; - while (true) { - const res = await request({ startDate, endDate, limit, offset }); + for (;;) { + const res = await request( + { startDate, endDate, limit, offset }, + analyticsDataClient + ); result.push(...res); offset += limit; if (res.length < limit) { @@ -45,11 +54,10 @@ export const saveGA4Data = async ( totalUsers, dateRange: `[${startDate}, ${endDate}]`, })); - const table = "article_ga4_data"; const updateRows = []; const insertRows = []; for (const { articleId, dateRange, totalUsers } of rows) { - const res = await knexRO(table) + const res = await knexRO(TABLE_NAME) .where({ articleId, dateRange }) .select("id", "totalUsers") .first(); @@ -64,11 +72,11 @@ export const saveGA4Data = async ( } if (updateRows.length > 0) { for (const { id, totalUsers } of updateRows) { - await knex(table).update({ totalUsers }).where({ id: id }); + await knex(TABLE_NAME).update({ totalUsers }).where({ id: id }); } } if (insertRows.length > 0) { - await knex(table).insert(insertRows); + await knex(TABLE_NAME).insert(insertRows); } }; @@ -117,18 +125,21 @@ const hashToId = async (hash: string) => { }; // https://developers.google.com/analytics/devguides/reporting/data/v1 -const request = async ({ - startDate, - endDate, - limit, - offset, -}: { - startDate: string; - endDate: string; - limit: number; - offset: number; -}): Promise => { - const [response] = await analyticsDataClient.runReport({ +const request = async ( + { + startDate, + endDate, + limit, + offset, + }: { + startDate: string; + endDate: string; + limit: number; + offset: number; + }, + client: BetaAnalyticsDataClient +): Promise => { + const [response] = await client.runReport({ property: `properties/${propertyId}`, dateRanges: [ { From 924e3ff205f4f257414db6ef71905b1b3a1fffd0 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Thu, 19 Oct 2023 21:02:24 +0800 Subject: [PATCH 05/15] feat(ga4): update script --- bin/sync-ga4-data.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/bin/sync-ga4-data.ts b/bin/sync-ga4-data.ts index c44d890..0aff1b8 100644 --- a/bin/sync-ga4-data.ts +++ b/bin/sync-ga4-data.ts @@ -1,15 +1,13 @@ import fs from "fs"; - -import { convertAndMerge, saveGA4Data } from "../lib/ga4.js"; - -// const data = JSON.parse(fs.readFileSync("data.json", "utf8")); -// convertAndMerge(data).then(console.log); - -saveGA4Data( - { "3": 1, "2": 4 }, - { startDate: "2021-10-29", endDate: "2021-10-29" } -).then(console.log); - -//fetchData({startDate: '2021-10-29', endDate: '2021-10-29'}).then(res => { -// fs.writeFileSync('data.json', JSON.stringify(res, null, 2)) -//}); +import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js"; + +const main = async () => { + const startDate = "2021-10-15"; + const endDate = "2023-10-19"; + const data = await fetchGA4Data({ startDate, endDate }); + fs.writeFileSync("./data.json", JSON.stringify(data)); + const convertedData = await convertAndMerge(data); + await saveGA4Data(convertedData, { startDate, endDate }); +}; + +main(); From fda6df4b9527f0174c3cdb1321856363dc0f8eb2 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 19 Oct 2023 21:36:22 +0800 Subject: [PATCH 06/15] feat(ga4): provide credentials from envs --- handlers/sync-ga4-data.ts | 4 ++++ lib/ga4.ts | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/handlers/sync-ga4-data.ts b/handlers/sync-ga4-data.ts index 478922b..5aed020 100644 --- a/handlers/sync-ga4-data.ts +++ b/handlers/sync-ga4-data.ts @@ -6,6 +6,10 @@ import { } from "../lib/ga4.js"; // envs +// MATTERS_GA4_PROPERTY_ID; +// MATTERS_GA4_PROJECT_ID; +// MATTERS_GA4_CLIENT_EMAIL; +// MATTERS_GA4_PRIVATE_KEY; // AWS EventBridge can configure the input event sent to Lambda, // see https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-transform-target-input.html for info. diff --git a/lib/ga4.ts b/lib/ga4.ts index f00b101..53ee375 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -3,6 +3,9 @@ import { BetaAnalyticsDataClient } from "@google-analytics/data"; import { pgKnexRO as knexRO, pgKnex as knex } from "./db.js"; const propertyId = process.env.MATTERS_GA4_PROPERTY_ID; +const projectId = process.env.MATTERS_GA4_PROJECT_ID; +const clientEmail = process.env.MATTERS_GA4_CLIENT_EMAIL; +const privateKey = process.env.MATTERS_GA4_PRIVATE_KEY || ""; export const TABLE_NAME = "article_ga4_data"; @@ -27,7 +30,13 @@ export const fetchGA4Data = async ({ startDate: string; endDate: string; }): Promise => { - const analyticsDataClient = new BetaAnalyticsDataClient(); + const analyticsDataClient = new BetaAnalyticsDataClient({ + projectId, + credentials: { + client_email: clientEmail, + private_key: privateKey.replace(/\\n/g, "\n"), + }, + }); const limit = 10000; let offset = 0; const result: Row[] = []; From c4820a12b8a59dee1ae8c4136e0572b9505ed447 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 19 Oct 2023 21:40:00 +0800 Subject: [PATCH 07/15] ci: fix envs --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca47d3d..ac290d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,6 @@ jobs: MATTERS_PG_PASSWORD: postgres MATTERS_PG_DATABASE: matters-test MATTERS_NEW_FEATURE_TAG_ID: 1 - MATTERS_PG_RO_CONNECTION_STRING: postgresql://postgres:postgres@localhost/test_matters-test + MATTERS_PG_RO_CONNECTION_STRING: postgresql://postgres:postgres@localhost/matters-test MATTERS_CACHE_HOST: localhost MATTERS_TEST_DB_SETUP: 1 From 900ae0b163c9d0996e2d52fb0160c29d209916c5 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:53:40 +0800 Subject: [PATCH 08/15] feat(ga4): add cloudformation config --- deployment/sync-ga-data.yml | 139 ++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 deployment/sync-ga-data.yml diff --git a/deployment/sync-ga-data.yml b/deployment/sync-ga-data.yml new file mode 100644 index 0000000..328ee14 --- /dev/null +++ b/deployment/sync-ga-data.yml @@ -0,0 +1,139 @@ +AWSTemplateFormatVersion: 2010-09-09 + +Description: lambda to sync ga article data + +Parameters: + imageUri: + Type: String + Description: "the Amazon ECR image URL of lambda-handlers to deploy" + mattersGA4PropertyId: + Type: String + mattersGA4ProjectId: + Type: String + mattersGA4ClientEmail: + Type: String + mattersGA4PrivateKey: + Type: String + mattersPgHost: + Type: String + mattersPgDatabase: + Type: String + mattersPgUser: + Type: String + mattersPgPassword: + Type: String + mattersPgRoConnectionString: + Type: String + +Resources: + Lambda: + Type: "AWS::Lambda::Function" + Properties: + Description: >- + A Lambda to sync ga articles data of matters.town. + Code: + ImageUri: !Ref imageUri + PackageType: Image + ImageConfig: + Command: + - sync-ga4-data.handler + Environment: + Variables: + MATTERS_GA4_PROPERTY_ID: !Ref mattersGA4PropertyId + MATTERS_GA4_PROJECT_ID: !Ref mattersGA4ProjectId + MATTERS_GA4_CLIENT_EMAIL: !Ref mattersGA4ClientEmail + MATTERS_GA4_PRIVATE_KEY: !Ref mattersGA4PrivateKey + MATTERS_PG_HOST: !Ref mattersPgHost + MATTERS_PG_DATABASE: !Ref mattersPgDatabase + MATTERS_PG_USER: !Ref mattersPgUser + MATTERS_PG_PASSWORD: !Ref mattersPgPassword + MATTERS_PG_RO_CONNECTION_STRING: !Ref mattersPgRoConnectionString + Architectures: + - x86_64 + MemorySize: 512 + Timeout: 900 + Role: !GetAtt LambdaRole.Arn + LambdaRole: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - "sts:AssumeRole" + Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + LambdaRolePolicy: + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref LambdaRole + PolicyName: "syncGaDataLambda" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: + - "logs:CreateLogGroup" + Effect: Allow + Resource: !Join + - "" + - - "arn:" + - !Ref "AWS::Partition" + - ":logs:" + - !Ref "AWS::Region" + - ":" + - !Ref "AWS::AccountId" + - ":*" + - Action: + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Effect: Allow + Resource: !Join + - "" + - - "arn:" + - !Ref "AWS::Partition" + - ":logs:" + - !Ref "AWS::Region" + - ":" + - !Ref "AWS::AccountId" + - ":log-group:/aws/lambda/" + - !Ref Lambda + - ":*" + CronEvent1: + Type: "AWS::Events::Rule" + Properties: + ScheduleExpression: "rate(5 minutes)" + Targets: + - Arn: !GetAtt Lambda.Arn + Id: CronEvent1LambdaTarget + Input: | + { + "type": "today" + } + CronEvent1Permission: + Type: "AWS::Lambda::Permission" + Properties: + Action: "lambda:InvokeFunction" + FunctionName: !Ref Lambda + Principal: events.amazonaws.com + SourceArn: !GetAtt CronEvent1.Arn + CronEvent2: + Type: "AWS::Events::Rule" + Properties: + ScheduleExpression: "cron(0 20 * * ? *)" + Targets: + - Arn: !GetAtt Lambda.Arn + Id: CronEvent2LambdaTarget + Input: | + { + "type": "yesterday" + } + CronEvent2Permission: + Type: "AWS::Lambda::Permission" + Properties: + Action: "lambda:InvokeFunction" + FunctionName: !Ref Lambda + Principal: events.amazonaws.com + SourceArn: !GetAtt CronEvent2.Arn From 4027269a2d755925805f7b3d37148a6c9e49a073 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:22:24 +0800 Subject: [PATCH 09/15] feat(ga4): minor --- README.md | 3 +++ handlers/sync-ga4-data.ts | 8 +++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 653212d..91faf45 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ feat(lambda): multiple lambda handlers in one repo - [x] user-retention - [x] likecoin - [x] cloudflare-image-stats-alert (not using docker, copy the source code to lambda to deploy) +- [x] sync-ga4-data ## Dependencies @@ -60,6 +61,8 @@ and push it with `docker image push {above-full-image}:v{date-tag}`, test it wit Can test trigger from the AWS Lambda Console +You can also deploy lambdas and related AWS resources by Cloudformation, see deployment/ folder. + ### Add manual auto deploy in Github Action: after first time manually create the Lambda function, all later updates can be done in Action CI: diff --git a/handlers/sync-ga4-data.ts b/handlers/sync-ga4-data.ts index 5aed020..546f40f 100644 --- a/handlers/sync-ga4-data.ts +++ b/handlers/sync-ga4-data.ts @@ -14,9 +14,7 @@ import { // AWS EventBridge can configure the input event sent to Lambda, // see https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-transform-target-input.html for info. type Event = { - data: { - type: "today" | "yesterday"; - }; + type: "today" | "yesterday"; }; const getDate = (type: "today" | "yesterday") => { @@ -28,8 +26,8 @@ const getDate = (type: "today" | "yesterday") => { }; export const handler = async (event: Event) => { - const { type } = event.data; - const startDate = getDate(type); + console.log("event: ", event); + const startDate = getDate(event.type); const endDate = startDate; const data = await fetchGA4Data({ startDate, endDate }); const convertedData = await convertAndMerge(data); From f85009376ea8cbb337912478b4e953d3e253537b Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:24:18 +0800 Subject: [PATCH 10/15] chore: bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 84515af..24294ad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lambda-handlers-image", - "version": "0.7.0", + "version": "0.8.0", "private": true, "type": "module", "scripts": { From 59694ac841cda38e7800a54955a6ac36f2f15fd1 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:51:01 +0800 Subject: [PATCH 11/15] fix(ga4): remove undefined key in `convertAndMerge` return --- lib/__test__/ga4.test.ts | 51 +++++++++++++++++++++++++++++++++++++++- lib/ga4.ts | 5 +++- package-lock.json | 4 ++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/lib/__test__/ga4.test.ts b/lib/__test__/ga4.test.ts index 7068e11..af74af6 100644 --- a/lib/__test__/ga4.test.ts +++ b/lib/__test__/ga4.test.ts @@ -1,5 +1,10 @@ import { pgKnex as knex } from "../db"; -import { saveGA4Data, TABLE_NAME, getLocalDateString } from "../ga4"; +import { + saveGA4Data, + TABLE_NAME, + getLocalDateString, + convertAndMerge, +} from "../ga4"; test("saveGA4Data", async () => { const startDate = "2021-01-01"; @@ -32,3 +37,47 @@ test("getLocalDateString", async () => { const dateStr = getLocalDateString(date); expect(dateStr).toBe("2021-01-01"); }); + +test("convertAndMerge", async () => { + const data = [ + { + path: "/@zeck_test_10/15911-未命名-bafybeiggtv7fcj5dci5x4hoogq7wzutortc3z2jyrsfzgdlwo7b4wjju4y", + totalUsers: "5", + }, + { + path: "/@alice_at_dev/21095-consequat-fugiat-aliqua-tempor-aliquip-bafybeieontjzxwnsu3qw6h6mumwcflaknftcwrkq6b2rpgrngtbuk5i3fu", + totalUsers: "2", + }, + { + path: "/@alice_at_dev/21098-laboris-ad-lorem-bafybeic6af5et4m4hot5avgu3g75lut4tnq4jzsa7fwi576bqwwf6ip7yq", + totalUsers: "2", + }, + { + path: "/@alice_at_dev/21099-ad-bafybeib2a5veytz6ze34aocqi4mjiyd23gvuzuh3r6teqhr2loomk3w2ji", + totalUsers: "2", + }, + { + path: "/@alice_at_dev/21099-ad-bafybeiby3td4oy2f473crb2bxrpad7fivhj4cfepslpay7mgssplwdojaa", + totalUsers: "2", + }, + { path: "/@alice_at_dev", totalUsers: "1" }, + { + path: "/@alice_at_dev/21094-amet-fugiat-commodo-pariatur-bafybeiffgowmxvnmdndqqptvpstu4a425scomyvh37koxy3ifind643sne", + totalUsers: "1", + }, + { + path: "/@alice_at_dev/21097-minim-laborum-tempor-ex-bafybeiawdwt7l5sdisdzmi4uma336dlzqahqavyjlmjti5xhlbmfbdwyhq", + totalUsers: "1", + }, + { path: "/@bob_at_dev", totalUsers: "1" }, + ]; + const result = await convertAndMerge(data); + expect(result).toStrictEqual({ + "15911": 5, + "21094": 1, + "21095": 2, + "21097": 1, + "21098": 2, + "21099": 4, + }); +}); diff --git a/lib/ga4.ts b/lib/ga4.ts index 53ee375..5bcfe51 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -101,13 +101,16 @@ export const convertAndMerge = async (rows: Row[]): Promise => { if (row.id in res) { res[row.id] += row.totalUsers; } else { - res[row.id] = row.totalUsers; + if (row.id) { + res[row.id] = row.totalUsers; + } } } return res; }; const pathToId = async (path: string) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars const [_, __, articlePath] = path.split("/"); if (articlePath) { const parts = articlePath.split("-"); diff --git a/package-lock.json b/package-lock.json index e245b51..bea50d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lambda-handlers-image", - "version": "0.7.0", + "version": "0.8.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "lambda-handlers-image", - "version": "0.7.0", + "version": "0.8.0", "dependencies": { "@aws-sdk/client-s3": "^3.267.0", "@aws-sdk/client-sqs": "^3.266.0", From 878d33bdcb1d9328d33ac907865ce589e3e6933c Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:37:08 +0800 Subject: [PATCH 12/15] feat(ga4): improve import script --- bin/sync-ga4-data.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/sync-ga4-data.ts b/bin/sync-ga4-data.ts index 0aff1b8..2ab746f 100644 --- a/bin/sync-ga4-data.ts +++ b/bin/sync-ga4-data.ts @@ -2,11 +2,25 @@ import fs from "fs"; import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js"; const main = async () => { - const startDate = "2021-10-15"; - const endDate = "2023-10-19"; - const data = await fetchGA4Data({ startDate, endDate }); - fs.writeFileSync("./data.json", JSON.stringify(data)); - const convertedData = await convertAndMerge(data); + const startDate = "2021-11-15"; + const endDate = "2023-11-15"; + const dataPath = `/tmp/lambda-handlers-sync-ga4-data(${startDate}-${endDate}).json`; + let data; + if (fs.existsSync(dataPath)) { + data = JSON.parse(fs.readFileSync(dataPath, "utf-8")); + } else { + data = await fetchGA4Data({ startDate, endDate }); + fs.writeFileSync(dataPath, JSON.stringify(data)); + } + const convertedPath = `/tmp/lambda-handlers-sync-ga4-data-converted(${startDate}-${endDate}).json`; + let convertedData; + if (fs.existsSync(convertedPath)) { + convertedData = JSON.parse(fs.readFileSync(convertedPath, "utf-8")); + } else { + convertedData = await fetchGA4Data({ startDate, endDate }); + fs.writeFileSync(convertedPath, JSON.stringify(data)); + } + convertedData = await convertAndMerge(data); await saveGA4Data(convertedData, { startDate, endDate }); }; From fdd6bdb589ae1c24b192836971705f0f63055576 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:21:44 +0800 Subject: [PATCH 13/15] fix(ga4): fix idlike check --- bin/sync-ga4-data.ts | 7 +++---- lib/ga4.ts | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/sync-ga4-data.ts b/bin/sync-ga4-data.ts index 2ab746f..b53b926 100644 --- a/bin/sync-ga4-data.ts +++ b/bin/sync-ga4-data.ts @@ -3,7 +3,7 @@ import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js"; const main = async () => { const startDate = "2021-11-15"; - const endDate = "2023-11-15"; + const endDate = "2023-11-14"; const dataPath = `/tmp/lambda-handlers-sync-ga4-data(${startDate}-${endDate}).json`; let data; if (fs.existsSync(dataPath)) { @@ -17,10 +17,9 @@ const main = async () => { if (fs.existsSync(convertedPath)) { convertedData = JSON.parse(fs.readFileSync(convertedPath, "utf-8")); } else { - convertedData = await fetchGA4Data({ startDate, endDate }); - fs.writeFileSync(convertedPath, JSON.stringify(data)); + convertedData = await convertAndMerge(data); + fs.writeFileSync(convertedPath, JSON.stringify(convertedData)); } - convertedData = await convertAndMerge(data); await saveGA4Data(convertedData, { startDate, endDate }); }; diff --git a/lib/ga4.ts b/lib/ga4.ts index 5bcfe51..2c93dca 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -116,7 +116,7 @@ const pathToId = async (path: string) => { const parts = articlePath.split("-"); const idLike = parts[0]; const hash = parts[parts.length - 1]; - if (!isNaN(parseInt(idLike))) { + if (/^-?\d+$/.test(idLike)) { return idLike; } else { return hashToId(hash); From cef9aba8162e001e2d3d017256c51d9c4e1f9cf5 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 16 Nov 2023 11:04:37 +0800 Subject: [PATCH 14/15] fix(ga4): check article id if valid --- lib/__test__/ga4.test.ts | 29 ++--------------------------- lib/ga4.ts | 4 +++- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/lib/__test__/ga4.test.ts b/lib/__test__/ga4.test.ts index af74af6..060fe47 100644 --- a/lib/__test__/ga4.test.ts +++ b/lib/__test__/ga4.test.ts @@ -41,43 +41,18 @@ test("getLocalDateString", async () => { test("convertAndMerge", async () => { const data = [ { - path: "/@zeck_test_10/15911-未命名-bafybeiggtv7fcj5dci5x4hoogq7wzutortc3z2jyrsfzgdlwo7b4wjju4y", + path: "/@zeck_test_10/1-未命名-bafybeiggtv7fcj5dci5x4hoogq7wzutortc3z2jyrsfzgdlwo7b4wjju4y", totalUsers: "5", }, - { - path: "/@alice_at_dev/21095-consequat-fugiat-aliqua-tempor-aliquip-bafybeieontjzxwnsu3qw6h6mumwcflaknftcwrkq6b2rpgrngtbuk5i3fu", - totalUsers: "2", - }, - { - path: "/@alice_at_dev/21098-laboris-ad-lorem-bafybeic6af5et4m4hot5avgu3g75lut4tnq4jzsa7fwi576bqwwf6ip7yq", - totalUsers: "2", - }, - { - path: "/@alice_at_dev/21099-ad-bafybeib2a5veytz6ze34aocqi4mjiyd23gvuzuh3r6teqhr2loomk3w2ji", - totalUsers: "2", - }, - { - path: "/@alice_at_dev/21099-ad-bafybeiby3td4oy2f473crb2bxrpad7fivhj4cfepslpay7mgssplwdojaa", - totalUsers: "2", - }, { path: "/@alice_at_dev", totalUsers: "1" }, { path: "/@alice_at_dev/21094-amet-fugiat-commodo-pariatur-bafybeiffgowmxvnmdndqqptvpstu4a425scomyvh37koxy3ifind643sne", totalUsers: "1", }, - { - path: "/@alice_at_dev/21097-minim-laborum-tempor-ex-bafybeiawdwt7l5sdisdzmi4uma336dlzqahqavyjlmjti5xhlbmfbdwyhq", - totalUsers: "1", - }, { path: "/@bob_at_dev", totalUsers: "1" }, ]; const result = await convertAndMerge(data); expect(result).toStrictEqual({ - "15911": 5, - "21094": 1, - "21095": 2, - "21097": 1, - "21098": 2, - "21099": 4, + "1": 5, }); }); diff --git a/lib/ga4.ts b/lib/ga4.ts index 2c93dca..3f054cd 100644 --- a/lib/ga4.ts +++ b/lib/ga4.ts @@ -97,11 +97,13 @@ export const convertAndMerge = async (rows: Row[]): Promise => { })) ); const res: MergedData = {}; + const ret = await knexRO("article").max("id").first(); + const maxLegalId = ret ? parseInt(ret.max) : 0; for (const row of await converted) { if (row.id in res) { res[row.id] += row.totalUsers; } else { - if (row.id) { + if (row.id && parseInt(row.id) <= maxLegalId) { res[row.id] = row.totalUsers; } } From ab4668c8a0410080a28c6246a24cf870d50b3ac9 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 16 Nov 2023 11:36:27 +0800 Subject: [PATCH 15/15] feat(ga4): add vpc config --- deployment/sync-ga-data.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deployment/sync-ga-data.yml b/deployment/sync-ga-data.yml index 328ee14..9ec179a 100644 --- a/deployment/sync-ga-data.yml +++ b/deployment/sync-ga-data.yml @@ -53,6 +53,12 @@ Resources: MemorySize: 512 Timeout: 900 Role: !GetAtt LambdaRole.Arn + VpcConfig: + SecurityGroupIds: + - sg-0adf0602441a6725f + SubnetIds: + - subnet-0b011dd1ca64fa0a1 + - subnet-0415147ddf68a48f2 LambdaRole: Type: "AWS::IAM::Role" Properties: