Add visitor table and poll clicks from Memphis (#599)

* wip: add tracker configuration

* feat: created dockerfile for tracker ms

* fix: prettier

* feat: remove unused deps

---------

Co-authored-by: orig <[email protected]>

.example.env

@@ -1,6 +1,7 @@
 # General
 BACKEND_APP_PORT=3000
 FRONTEND_APP_PORT=5000
+TRACKER_APP_PORT=3001
 NODE_ENV=development
 
 # RATE LIMIT
@@ -26,10 +27,6 @@ REDIS_PORT=6379
 REDIS_PASSWORD=password
 REDIS_TTL=1800  # 30 minutes in seconds (default ttl)
 
-# AUTH
-JWT_ACCESS_SECRET=abc1234
-JWT_REFRESH_SECRET=abc1234
-
 # NOVU - You don't need this when running locally (just verify your email from the database)
 NOVU_API_KEY=Get it from https://novu.co/
 
@@ -42,5 +39,11 @@ MEMPHIS_PASSWORD=Get it from https://memphis.dev/
 MEMPHIS_ACCOUNT_ID=Get it from https://memphis.dev/
 
 # AUTH
+AUTH_JWT_ACCESS_SECRET=abc1234
+AUTH_JWT_REFRESH_SECRET=abc1234
+
 AUTH_GOOGLE_CLIENT_ID=Get it from https://console.cloud.google.com/apis/credentials
 AUTH_GOOGLE_CLIENT_SECRET=Get it from https://console.cloud.google.com/apis/credentials
+
+# TRACKER
+TRACKER_STATS_QUEUE_NAME=stats

.vscode/launch.json

@@ -9,6 +9,15 @@
       "runtimeArgs": ["nx", "serve", "backend"],
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}/apps/backend"
+    },
+    {
+      "name": "Debug tracker",
+      "request": "launch",
+      "type": "node",
+      "runtimeExecutable": "npx",
+      "runtimeArgs": ["nx", "serve", "tracker"],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/apps/tracker"
     }
   ]
 }

apps/backend/src/auth/auth.controller.spec.ts

@@ -38,10 +38,13 @@ describe('AuthController', () => {
   const MOCK_TOKENS = { accessToken: 'access_token', refreshToken: 'refresh_token' };
   const MOCK_CONFIG: Partial<Configuration> = {
     front: { domain: 'example.com', apiDomain: 'http://localhost:3000', clientSideApiDomain: 'http://localhost:3000' },
-    general: { env: 'production', backendPort: 3000, frontendPort: 5000 },
-    jwt: {
-      accessSecret: 'secret',
-      refreshSecret: 'secret',
+    general: { env: 'production', backendPort: 3000, frontendPort: 5000, trackerPort: 3001 },
+    auth: {
+      jwt: {
+        accessSecret: 'secret',
+        refreshSecret: 'secret',
+      },
+      google: {} as any,
     },
   };
 

apps/backend/src/auth/auth.module.ts

@@ -22,7 +22,7 @@ import { UsersModule } from '../core/users/users.module';
     JwtModule.registerAsync({
       inject: [AppConfigService],
       useFactory: (config: AppConfigService) => ({
-        secret: config.getConfig().jwt.accessSecret,
+        secret: config.getConfig().auth.jwt.accessSecret,
         signOptions: { expiresIn: '5m' },
       }),
     }),

apps/backend/src/auth/auth.service.ts

@@ -149,7 +149,7 @@ export class AuthService {
   async generateTokens(user: UserContext): Promise<{ accessToken: string; refreshToken: string }> {
     const tokens = {
       accessToken: this.generateToken(user),
-      refreshToken: this.generateToken(user, '7d', this.appConfigService.getConfig().jwt.refreshSecret),
+      refreshToken: this.generateToken(user, '7d', this.appConfigService.getConfig().auth.jwt.refreshSecret),
     };
 
     await this.prisma.user.update({

apps/backend/src/auth/strategies/jwt-refresh.strategy.ts

@@ -15,7 +15,7 @@ export class JwtRefreshStrategy extends PassportStrategy(Strategy, 'jwt-refresh'
         },
       ]),
       ignoreExpiration: false,
-      secretOrKey: appConfigService.getConfig().jwt.refreshSecret,
+      secretOrKey: appConfigService.getConfig().auth.jwt.refreshSecret,
       passReqToCallback: true,
     });
   }

apps/backend/src/auth/strategies/jwt.strategy.ts

@@ -21,7 +21,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
         },
       ]),
       ignoreExpiration: false,
-      secretOrKey: appConfigService.getConfig().jwt.accessSecret,
+      secretOrKey: appConfigService.getConfig().auth.jwt.accessSecret,
     });
   }
 

apps/backend/src/auth/strategies/verify.strategy.ts

@@ -9,7 +9,7 @@ export class VerifyStrategy extends PassportStrategy(Strategy, 'verify') {
     super({
       jwtFromRequest: ExtractJwt.fromHeader('token'),
       ignoreExpiration: false,
-      secretOrKey: appConfigService.getConfig().jwt.accessSecret,
+      secretOrKey: appConfigService.getConfig().auth.jwt.accessSecret,
     });
   }
 

apps/backend/src/main.ts

@@ -5,6 +5,7 @@ import { useContainer } from 'class-validator';
 import cookieParser from 'cookie-parser';
 import { AppModule } from './app.module';
 import { AppConfigService } from '@reduced.to/config';
+import { AppLoggerSerivce } from '@reduced.to/logger';
 
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule);
@@ -23,8 +24,11 @@ async function bootstrap() {
   useContainer(app.select(AppModule), { fallbackOnErrors: true });
 
   const port = app.get(AppConfigService).getConfig().general.backendPort;
+  const logger = app.get(AppLoggerSerivce);
 
   await app.listen(port);
+
+  logger.log(`Starting backend on port ${port}`);
 }
 
 bootstrap();

apps/backend/src/shortener/producer/shortener.producer.ts

@@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import { ProducerService } from '@reduced.to/queue-manager';
+import { AppConfigService } from '@reduced.to/config';
 
 const SHORTENER_PRODUCER_NAME = 'shortener';
-const SHORTENER_QUEUE_NAME = 'stats';
 
 @Injectable()
 export class ShortenerProducer extends ProducerService {
-  constructor() {
-    super(SHORTENER_PRODUCER_NAME, SHORTENER_QUEUE_NAME);
+  constructor(config: AppConfigService) {
+    super(SHORTENER_PRODUCER_NAME, config.getConfig().tracker.stats.queueName);
   }
 }

apps/tracker/.dockerignore

@@ -0,0 +1,8 @@
+node_modules
+dist
+npm-debug.log
+CONTRIBUTE.MD
+README.md
+.gitignore
+LICENSE
+.DS_Store

apps/tracker/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

apps/tracker/Dockerfile

@@ -0,0 +1,47 @@
+# --------------------------------------------
+# Dependencies Stage
+# --------------------------------------------
+FROM node:19.2-alpine3.15 as dependencies
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN apk add --update python3 make g++\
+   && rm -rf /var/cache/apk/*
+
+RUN npm ci
+
+# --------------------------------------------
+# Build Stage
+# --------------------------------------------
+# Intermediate docker image to build the bundle in and install dependencies
+FROM node:19.2-alpine3.15 as build
+WORKDIR /app
+
+COPY . .
+COPY --from=dependencies /app/node_modules ./node_modules
+
+# Run prisma generate & build the bundle in production mode
+RUN npx nx build tracker --prod --skip-nx-cache
+
+# --------------------------------------------
+# Production Stage
+# --------------------------------------------
+FROM node:19.2-alpine3.15 as production
+WORKDIR /app
+
+COPY --from=build /app/dist/apps/tracker ./tracker
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/libs/ ./libs
+
+EXPOSE 3001
+
+# Start the application
+CMD sh -c "npx nx migrate-deploy prisma && node backend/main.js"
+
+
+
+
+
+
+

apps/tracker/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: 'tracker',
+  preset: '../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../coverage/apps/tracker',
+};

apps/tracker/project.json

@@ -0,0 +1,78 @@
+{
+  "name": "tracker",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/tracker/src",
+  "projectType": "application",
+  "targets": {
+    "build": {
+      "executor": "@nx/webpack:webpack",
+      "outputs": ["{options.outputPath}"],
+      "defaultConfiguration": "production",
+      "options": {
+        "target": "node",
+        "compiler": "tsc",
+        "outputPath": "dist/apps/tracker",
+        "main": "apps/tracker/src/main.ts",
+        "tsConfig": "apps/tracker/tsconfig.app.json",
+        "isolatedConfig": true,
+        "webpackConfig": "apps/tracker/webpack.config.js"
+      },
+      "configurations": {
+        "development": {},
+        "production": {}
+      }
+    },
+    "serve": {
+      "executor": "@nx/js:node",
+      "defaultConfiguration": "development",
+      "options": {
+        "buildTarget": "tracker:build"
+      },
+      "configurations": {
+        "development": {
+          "buildTarget": "tracker:build:development"
+        },
+        "production": {
+          "buildTarget": "tracker:build:production"
+        }
+      }
+    },
+    "lint": {
+      "executor": "@nx/eslint:lint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": ["apps/tracker/**/*.ts"]
+      }
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "apps/tracker/jest.config.ts",
+        "passWithNoTests": true
+      },
+      "configurations": {
+        "ci": {
+          "ci": true,
+          "codeCoverage": true
+        }
+      }
+    },
+    "docker-build": {
+      "dependsOn": ["build"],
+      "command": "docker build -f apps/tracker/Dockerfile . -t tracker"
+    },
+    "push-image-to-registry": {
+      "dependsOn": ["docker-build"],
+      "executor": "nx:run-commands",
+      "options": {
+        "commands": [
+          "docker image tag tracker ghcr.io/{args.repository}/tracker:master",
+          "docker push ghcr.io/{args.repository}/tracker:master"
+        ],
+        "parallel": false
+      }
+    }
+  },
+  "tags": []
+}

apps/tracker/src/app/app.module.ts

@@ -0,0 +1,10 @@
+import { Global, Module } from '@nestjs/common';
+import { AppConfigModule } from '@reduced.to/config';
+import { AppLoggerModule } from '@reduced.to/logger';
+import { StatsModule } from '../stats/stats.module';
+
+@Global()
+@Module({
+  imports: [AppConfigModule, AppLoggerModule, StatsModule],
+})
+export class AppModule {}

apps/tracker/src/main.ts

@@ -0,0 +1,27 @@
+import { VersioningType } from '@nestjs/common';
+import { NestFactory } from '@nestjs/core';
+import { NestExpressApplication } from '@nestjs/platform-express';
+import { AppConfigService } from '@reduced.to/config';
+import { AppModule } from './app/app.module';
+import { AppLoggerSerivce } from '@reduced.to/logger';
+
+async function bootstrap() {
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
+
+  app.enableVersioning({
+    type: VersioningType.URI,
+    defaultVersion: '1',
+    prefix: 'api/v',
+  });
+
+  app.enableCors({ origin: true, credentials: true });
+
+  const port = app.get(AppConfigService).getConfig().general.trackerPort;
+  const logger = app.get(AppLoggerSerivce);
+
+  await app.listen(port);
+
+  logger.log(`Starting tracker on port ${port}`);
+}
+
+bootstrap();

apps/tracker/src/stats/stats.consumer.ts

@@ -0,0 +1,36 @@
+import { Injectable } from '@nestjs/common';
+import { ConsumerService } from '@reduced.to/queue-manager';
+import { AppConfigService } from '@reduced.to/config';
+import { AppLoggerSerivce } from '@reduced.to/logger';
+import { StatsService } from './stats.service';
+import * as geoip from 'geoip-lite';
+import { createHash } from 'node:crypto';
+import { Message } from 'memphis-dev/*';
+
+@Injectable()
+export class StatsConsumer extends ConsumerService {
+  constructor(config: AppConfigService, private readonly loggerService: AppLoggerSerivce, private readonly statsService: StatsService) {
+    super('tracker', config.getConfig().tracker.stats.queueName);
+  }
+
+  async onMessage(message: Message): Promise<void> {
+    const { ip, userAgent, key, url } = message.getDataAsJson() as { ip: string; userAgent: string; key: string; url: string };
+    message.ack();
+
+    const hashedIp = createHash('sha256').update(ip).digest('hex');
+    const isUniqueVisit = await this.statsService.isUniqueVisit(key, hashedIp);
+
+    if (!isUniqueVisit) {
+      return;
+    }
+
+    const geo = geoip.lookup(ip);
+    await this.statsService.addVisit(key, {
+      hashedIp,
+      ua: userAgent,
+      geo,
+    });
+
+    this.loggerService.log(`Added unique visit for ${key}`);
+  }
+}