Merge pull request #93 from thematters/feat/sync-ga-data

feat(ga): sync ga data lambda

.github/workflows/test.yml

@@ -50,7 +50,6 @@ jobs:
           MATTERS_PG_PASSWORD: postgres
           MATTERS_PG_DATABASE: matters-test
           MATTERS_NEW_FEATURE_TAG_ID: 1
-          MATTERS_PG_RO_CONNECTION_STRING: postgresql://postgres:postgres@localhost/test_matters-test
+          MATTERS_PG_RO_CONNECTION_STRING: postgresql://postgres:postgres@localhost/matters-test
           MATTERS_CACHE_HOST: localhost
-
-
+          MATTERS_TEST_DB_SETUP: 1

README.md

@@ -13,6 +13,7 @@ feat(lambda): multiple lambda handlers in one repo
 - [x] user-retention
 - [x] likecoin
 - [x] cloudflare-image-stats-alert (not using docker, copy the source code to lambda to deploy)
+- [x] sync-ga4-data
 
 ## Dependencies
 
@@ -60,6 +61,8 @@ and push it with `docker image push {above-full-image}:v{date-tag}`, test it wit
 
 Can test trigger from the AWS Lambda Console
 
+You can also deploy lambdas and related AWS resources by Cloudformation, see deployment/ folder.
+
 ### Add manual auto deploy in Github Action:
 
 after first time manually create the Lambda function, all later updates can be done in Action CI:

bin/sync-ga4-data.ts

@@ -0,0 +1,26 @@
+import fs from "fs";
+import { fetchGA4Data, convertAndMerge, saveGA4Data } from "../lib/ga4.js";
+
+const main = async () => {
+  const startDate = "2021-11-15";
+  const endDate = "2023-11-14";
+  const dataPath = `/tmp/lambda-handlers-sync-ga4-data(${startDate}-${endDate}).json`;
+  let data;
+  if (fs.existsSync(dataPath)) {
+    data = JSON.parse(fs.readFileSync(dataPath, "utf-8"));
+  } else {
+    data = await fetchGA4Data({ startDate, endDate });
+    fs.writeFileSync(dataPath, JSON.stringify(data));
+  }
+  const convertedPath = `/tmp/lambda-handlers-sync-ga4-data-converted(${startDate}-${endDate}).json`;
+  let convertedData;
+  if (fs.existsSync(convertedPath)) {
+    convertedData = JSON.parse(fs.readFileSync(convertedPath, "utf-8"));
+  } else {
+    convertedData = await convertAndMerge(data);
+    fs.writeFileSync(convertedPath, JSON.stringify(convertedData));
+  }
+  await saveGA4Data(convertedData, { startDate, endDate });
+};
+
+main();

deployment/sync-ga-data.yml

@@ -0,0 +1,145 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Description: lambda to sync ga article data
+
+Parameters:
+  imageUri:
+    Type: String
+    Description: "the Amazon ECR image URL of lambda-handlers to deploy"
+  mattersGA4PropertyId:
+    Type: String
+  mattersGA4ProjectId:
+    Type: String
+  mattersGA4ClientEmail:
+    Type: String
+  mattersGA4PrivateKey:
+    Type: String
+  mattersPgHost:
+    Type: String
+  mattersPgDatabase:
+    Type: String
+  mattersPgUser:
+    Type: String
+  mattersPgPassword:
+    Type: String
+  mattersPgRoConnectionString:
+    Type: String
+
+Resources:
+  Lambda:
+    Type: "AWS::Lambda::Function"
+    Properties:
+      Description: >-
+        A Lambda to sync ga articles data of matters.town.
+      Code:
+        ImageUri: !Ref imageUri
+      PackageType: Image
+      ImageConfig:
+        Command:
+          - sync-ga4-data.handler
+      Environment:
+        Variables:
+          MATTERS_GA4_PROPERTY_ID: !Ref mattersGA4PropertyId
+          MATTERS_GA4_PROJECT_ID: !Ref mattersGA4ProjectId
+          MATTERS_GA4_CLIENT_EMAIL: !Ref mattersGA4ClientEmail
+          MATTERS_GA4_PRIVATE_KEY: !Ref mattersGA4PrivateKey
+          MATTERS_PG_HOST: !Ref mattersPgHost
+          MATTERS_PG_DATABASE: !Ref mattersPgDatabase
+          MATTERS_PG_USER: !Ref mattersPgUser
+          MATTERS_PG_PASSWORD: !Ref mattersPgPassword
+          MATTERS_PG_RO_CONNECTION_STRING: !Ref mattersPgRoConnectionString
+      Architectures:
+        - x86_64
+      MemorySize: 512
+      Timeout: 900
+      Role: !GetAtt LambdaRole.Arn
+      VpcConfig:
+        SecurityGroupIds:
+          - sg-0adf0602441a6725f
+        SubnetIds:
+          - subnet-0b011dd1ca64fa0a1
+          - subnet-0415147ddf68a48f2
+  LambdaRole:
+    Type: "AWS::IAM::Role"
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Action:
+              - "sts:AssumeRole"
+            Effect: Allow
+            Principal:
+              Service:
+                - lambda.amazonaws.com
+  LambdaRolePolicy:
+    Type: "AWS::IAM::Policy"
+    Properties:
+      Roles:
+        - !Ref LambdaRole
+      PolicyName: "syncGaDataLambda"
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Action:
+              - "logs:CreateLogGroup"
+            Effect: Allow
+            Resource: !Join
+              - ""
+              - - "arn:"
+                - !Ref "AWS::Partition"
+                - ":logs:"
+                - !Ref "AWS::Region"
+                - ":"
+                - !Ref "AWS::AccountId"
+                - ":*"
+          - Action:
+              - "logs:CreateLogStream"
+              - "logs:PutLogEvents"
+            Effect: Allow
+            Resource: !Join
+              - ""
+              - - "arn:"
+                - !Ref "AWS::Partition"
+                - ":logs:"
+                - !Ref "AWS::Region"
+                - ":"
+                - !Ref "AWS::AccountId"
+                - ":log-group:/aws/lambda/"
+                - !Ref Lambda
+                - ":*"
+  CronEvent1:
+    Type: "AWS::Events::Rule"
+    Properties:
+      ScheduleExpression: "rate(5 minutes)"
+      Targets:
+        - Arn: !GetAtt Lambda.Arn
+          Id: CronEvent1LambdaTarget
+          Input: |
+            {
+              "type": "today"
+            }
+  CronEvent1Permission:
+    Type: "AWS::Lambda::Permission"
+    Properties:
+      Action: "lambda:InvokeFunction"
+      FunctionName: !Ref Lambda
+      Principal: events.amazonaws.com
+      SourceArn: !GetAtt CronEvent1.Arn
+  CronEvent2:
+    Type: "AWS::Events::Rule"
+    Properties:
+      ScheduleExpression: "cron(0 20 * * ? *)"
+      Targets:
+        - Arn: !GetAtt Lambda.Arn
+          Id: CronEvent2LambdaTarget
+          Input: |
+            {
+              "type": "yesterday"
+            }
+  CronEvent2Permission:
+    Type: "AWS::Lambda::Permission"
+    Properties:
+      Action: "lambda:InvokeFunction"
+      FunctionName: !Ref Lambda
+      Principal: events.amazonaws.com
+      SourceArn: !GetAtt CronEvent2.Arn

handlers/sync-ga4-data.ts

@@ -0,0 +1,35 @@
+import {
+  fetchGA4Data,
+  convertAndMerge,
+  saveGA4Data,
+  getLocalDateString,
+} from "../lib/ga4.js";
+
+// envs
+// MATTERS_GA4_PROPERTY_ID;
+// MATTERS_GA4_PROJECT_ID;
+// MATTERS_GA4_CLIENT_EMAIL;
+// MATTERS_GA4_PRIVATE_KEY;
+
+// AWS EventBridge can configure the input event sent to Lambda,
+// see https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-transform-target-input.html for info.
+type Event = {
+  type: "today" | "yesterday";
+};
+
+const getDate = (type: "today" | "yesterday") => {
+  const date = new Date();
+  if (type === "yesterday") {
+    date.setDate(date.getDate() - 1);
+  }
+  return getLocalDateString(date);
+};
+
+export const handler = async (event: Event) => {
+  console.log("event: ", event);
+  const startDate = getDate(event.type);
+  const endDate = startDate;
+  const data = await fetchGA4Data({ startDate, endDate });
+  const convertedData = await convertAndMerge(data);
+  await saveGA4Data(convertedData, { startDate, endDate });
+};

jest.config.cjs

@@ -6,7 +6,6 @@ module.exports = {
   testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$',
   testPathIgnorePatterns: ['/node_modules/', '/matters-server/'],
   globalSetup: '<rootDir>/matters-server/db/globalTestSetup.js',
-  globalTeardown: '<rootDir>/matters-server/db/globalTestTeardown.js',
   transform: {
   '\\.tsx?$': ['ts-jest', {
     useESM: true,

lib/__test__/ga4.test.ts

@@ -0,0 +1,58 @@
+import { pgKnex as knex } from "../db";
+import {
+  saveGA4Data,
+  TABLE_NAME,
+  getLocalDateString,
+  convertAndMerge,
+} from "../ga4";
+
+test("saveGA4Data", async () => {
+  const startDate = "2021-01-01";
+  const endDate = "2021-01-01";
+
+  // insert
+  await saveGA4Data({ "1": 1, "2": 2 }, { startDate, endDate });
+
+  const rows = await knex(TABLE_NAME)
+    .select("*")
+    .where({ dateRange: `[${startDate}, ${endDate}]` });
+  expect(rows.length).toBe(2);
+
+  // insert and update
+  await saveGA4Data({ "1": 2, "3": 3 }, { startDate, endDate });
+
+  const rows2 = await knex(TABLE_NAME)
+    .select("*")
+    .where({ dateRange: `[${startDate}, ${endDate}]` });
+  expect(rows2.length).toBe(3);
+  for (const row of rows2) {
+    if (row.articleId === "1") {
+      expect(row.totalUsers).toBe("2");
+    }
+  }
+});
+
+test("getLocalDateString", async () => {
+  const date = new Date("2021-01-01");
+  const dateStr = getLocalDateString(date);
+  expect(dateStr).toBe("2021-01-01");
+});
+
+test("convertAndMerge", async () => {
+  const data = [
+    {
+      path: "/@zeck_test_10/1-未命名-bafybeiggtv7fcj5dci5x4hoogq7wzutortc3z2jyrsfzgdlwo7b4wjju4y",
+      totalUsers: "5",
+    },
+    { path: "/@alice_at_dev", totalUsers: "1" },
+    {
+      path: "/@alice_at_dev/21094-amet-fugiat-commodo-pariatur-bafybeiffgowmxvnmdndqqptvpstu4a425scomyvh37koxy3ifind643sne",
+      totalUsers: "1",
+    },
+    { path: "/@bob_at_dev", totalUsers: "1" },
+  ];
+  const result = await convertAndMerge(data);
+  expect(result).toStrictEqual({
+    "1": 5,
+  });
+});

lib/db.ts

@@ -5,12 +5,10 @@ import { getKnexClient, getPostgresJsClient } from "./utils/db.js";
 const CLOUDFLARE_IMAGE_ENDPOINT = process.env.CLOUDFLARE_IMAGE_ENDPOINT || "";
 const MATTERS_AWS_S3_ENDPOINT = process.env.MATTERS_AWS_S3_ENDPOINT || "";
 
-const isTest = process.env.MATTERS_ENV === "test";
 const dbHost = process.env.MATTERS_PG_HOST || "";
 const dbUser = process.env.MATTERS_PG_USER || "";
 const dbPasswd = process.env.MATTERS_PG_PASSWORD || "";
-const _dbName = process.env.MATTERS_PG_DATABASE || "";
-const dbName = isTest ? "test_" + _dbName : _dbName;
+const dbName = process.env.MATTERS_PG_DATABASE || "";
 
 const databaseURL =
   process.env.PG_CONNECTION_STRING ||