BC-7261 - board websocket metrics

ansible/roles/schulcloud-server-core/tasks/main.yml

@@ -322,3 +322,10 @@
       template: board-collaboration-ingress.yml.j2
       apply: yes
       state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
+
+  - name: BoardCollaborationServiceMonitor
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: '{{ NAMESPACE }}'
+      template: board-collaboration-svc-monitor.yml.j2
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"

ansible/roles/schulcloud-server-core/templates/board-collaboration-deployment.yml.j2

@@ -49,9 +49,9 @@ spec:
         - containerPort: 4450
           name: websocket
           protocol: TCP
-        # - containerPort: 9090
-        #  name: api-metrics
-        #  protocol: TCP
+        - containerPort: 9090
+          name: api-metrics
+          protocol: TCP
         envFrom:
         - configMapRef:
             name: api-configmap

ansible/roles/schulcloud-server-core/templates/board-collaboration-service.yml.j2

@@ -13,9 +13,9 @@ spec:
       targetPort: 4450
       protocol: TCP
       name: websocket
-    #- port: {{ PORT_METRICS_SERVER }}
-    #  targetPort: 9090 # TODO
-    #  protocol: TCP
-    #  name: api-metrics
+    - port: {{ PORT_METRICS_SERVER }}
+      targetPort: 9090 # TODO
+      protocol: TCP
+      name: api-metrics
   selector:
     app: board-collaboration

ansible/roles/schulcloud-server-core/templates/board-collaboration-svc-monitor.yml.j2

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: board-collaboration-svc-monitor
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: board-collaboration
+spec:
+  selector:
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: in
+        values:
+          - board-collaboration-svc
+  endpoints:
+  - path: /metrics
+    port: api-metrics

apps/server/src/apps/board-collaboration.app.ts

@@ -8,43 +8,50 @@ import { install as sourceMapInstall } from 'source-map-support';
 // application imports
 import { SwaggerDocumentOptions } from '@nestjs/swagger';
 import { DB_URL } from '@src/config';
-import { LegacyLogger } from '@src/core/logger';
+import { LegacyLogger, Logger } from '@src/core/logger';
 import { MongoIoAdapter } from '@src/infra/socketio';
 import { BoardCollaborationModule } from '@src/modules/board/board-collaboration.module';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
+import express from 'express';
+import {
+	addPrometheusMetricsMiddlewaresIfEnabled,
+	createAndStartPrometheusMetricsAppIfEnabled,
+} from '@src/apps/helpers/prometheus-metrics';
+import { ExpressAdapter } from '@nestjs/platform-express';
 
 async function bootstrap() {
 	sourceMapInstall();
 
-	const nestApp = await NestFactory.create(BoardCollaborationModule);
-
-	// WinstonLogger
+	const nestExpress = express();
+	const nestExpressAdapter = new ExpressAdapter(nestExpress);
+	const nestApp = await NestFactory.create(BoardCollaborationModule, nestExpressAdapter);
 	nestApp.useLogger(await nestApp.resolve(LegacyLogger));
-
-	// customize nest app settings
 	nestApp.enableCors({ exposedHeaders: ['Content-Disposition'] });
 
 	const mongoIoAdapter = new MongoIoAdapter(nestApp);
 	await mongoIoAdapter.connectToMongoDb(DB_URL);
-
 	nestApp.useWebSocketAdapter(mongoIoAdapter);
 
 	const options: SwaggerDocumentOptions = {
 		operationIdFactory: (_controllerKey: string, methodKey: string) => methodKey,
 	};
 	enableOpenApiDocs(nestApp, 'docs', options);
+	const logger = await nestApp.resolve(Logger);
 
 	await nestApp.init();
 
+	addPrometheusMetricsMiddlewaresIfEnabled(logger, nestExpress);
 	const port = 4450;
 	const basePath = '/board-collaboration';
 
 	nestApp.setGlobalPrefix(basePath);
-	await nestApp.listen(port);
+	await nestApp.listen(port, () => {
+		createAndStartPrometheusMetricsAppIfEnabled(logger);
+	});
 
 	console.log('##########################################');
 	console.log(`### Start Board Collaboration Server   ###`);
-	console.log(`### Port:      ${port}                     ###`);
+	console.log(`### Port:      ${port}                 ###`);
 	console.log(`### Base path: ${basePath}             ###`);
 	console.log('##########################################');
 }

apps/server/src/modules/authentication/index.ts

@@ -3,4 +3,5 @@ export { Authenticate, CurrentUser, JWT } from './decorator';
 export { ICurrentUser } from './interface';
 export { AuthenticationService } from './services';
 export { XApiKeyConfig } from './config';
+export { WsJwtAuthGuard } from './guard/ws-jwt-auth.guard';
 export { AuthenticationConfig } from './authentication-config';

apps/server/src/modules/board/board-ws-api.module.ts

@@ -1,15 +1,26 @@
 import { forwardRef, Module } from '@nestjs/common';
 import { CourseRepo } from '@shared/repo';
 import { LoggerModule } from '@src/core/logger';
-import { AuthorizationModule } from '../authorization';
+import { AuthorizationModule } from '@modules/authorization';
+import { UserModule } from '@modules/user';
 import { BoardModule } from './board.module';
 import { BoardCollaborationGateway } from './gateway/board-collaboration.gateway';
+import { MetricsService } from './metrics/metrics.service';
 import { BoardUc, CardUc, ColumnUc, ElementUc } from './uc';
 import { BoardNodePermissionService } from './service';
 
 @Module({
-	imports: [BoardModule, forwardRef(() => AuthorizationModule), LoggerModule],
-	providers: [BoardCollaborationGateway, BoardNodePermissionService, CardUc, ColumnUc, ElementUc, BoardUc, CourseRepo],
+	imports: [BoardModule, forwardRef(() => AuthorizationModule), LoggerModule, UserModule],
+	providers: [
+		BoardCollaborationGateway,
+		BoardNodePermissionService,
+		CardUc,
+		ColumnUc,
+		ElementUc,
+		BoardUc,
+		CourseRepo,
+		MetricsService,
+	],
 	exports: [],
 })
 export class BoardWsApiModule {}

apps/server/src/modules/board/gateway/board-collaboration.gateway.ts

@@ -1,14 +1,23 @@
-import { WsValidationPipe, Socket } from '@infra/socketio';
+import { Socket, WsValidationPipe } from '@infra/socketio';
 import { MikroORM, UseRequestContext } from '@mikro-orm/core';
+import { WsJwtAuthGuard } from '@modules/authentication';
 import { UseGuards, UsePipes } from '@nestjs/common';
-import { SubscribeMessage, WebSocketGateway, WsException } from '@nestjs/websockets';
-import { WsJwtAuthGuard } from '@modules/authentication/guard/ws-jwt-auth.guard';
+import {
+	OnGatewayDisconnect,
+	SubscribeMessage,
+	WebSocketGateway,
+	WebSocketServer,
+	WsException,
+} from '@nestjs/websockets';
+import { Server } from 'socket.io';
 import {
 	BoardResponseMapper,
 	CardResponseMapper,
 	ColumnResponseMapper,
 	ContentElementResponseFactory,
 } from '../controller/mapper';
+import { MetricsService } from '../metrics/metrics.service';
+import { TrackExecutionTime } from '../metrics/track-execution-time.decorator';
 import { BoardNodeAuthorizableService } from '../service';
 import { BoardUc, CardUc, ColumnUc, ElementUc } from '../uc';
 import {
@@ -36,22 +45,46 @@ import { UpdateContentElementMessageParams } from './dto/update-content-element.
 @UsePipes(new WsValidationPipe())
 @WebSocketGateway(BoardCollaborationConfiguration.websocket)
 @UseGuards(WsJwtAuthGuard)
-export class BoardCollaborationGateway {
+export class BoardCollaborationGateway implements OnGatewayDisconnect {
+	@WebSocketServer()
+	server!: Server;
+
+	// TODO: use loggables instead of legacy logger
 	constructor(
 		private readonly orm: MikroORM,
 		private readonly boardUc: BoardUc,
 		private readonly columnUc: ColumnUc,
 		private readonly cardUc: CardUc,
 		private readonly elementUc: ElementUc,
+		private readonly metricsService: MetricsService,
 		private readonly authorizableService: BoardNodeAuthorizableService // to be removed
 	) {}
 
+	trackExecutionTime(methodName: string, executionTimeMs: number) {
+		if (this.metricsService) {
+			this.metricsService.setExecutionTime(methodName, executionTimeMs);
+		}
+	}
+
 	private getCurrentUser(socket: Socket) {
 		const { user } = socket.handshake;
 		if (!user) throw new WsException('Not Authenticated.');
 		return user;
 	}
 
+	private async updateRoomsAndUsersMetrics(socket: Socket) {
+		const roomCount = Array.from(this.server.of('/').adapter.rooms.keys()).filter((key) =>
+			key.startsWith('board_')
+		).length;
+		this.metricsService.setNumberOfBoardRooms(roomCount);
+		const { user } = socket.handshake;
+		await this.metricsService.trackRoleOfClient(socket.id, user?.userId);
+	}
+
+	public handleDisconnect(socket: Socket): void {
+		this.metricsService.untrackClient(socket.id);
+	}
+
 	@SubscribeMessage('delete-board-request')
 	@UseRequestContext()
 	async deleteBoard(socket: Socket, data: DeleteBoardMessageParams) {
@@ -64,9 +97,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-board-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateBoardTitle(socket: Socket, data: UpdateBoardTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.boardId, action: 'update-board-title' });
@@ -78,9 +113,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-card-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateCardTitle(socket: Socket, data: UpdateCardTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.cardId, action: 'update-card-title' });
@@ -92,6 +129,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-card-height-request')
@@ -106,6 +144,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-card-request')
@@ -120,9 +159,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-card-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async createCard(socket: Socket, data: CreateCardMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.columnId, action: 'create-card' });
@@ -140,6 +181,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-column-request')
@@ -167,6 +209,7 @@ export class BoardCollaborationGateway {
 	}
 
 	@SubscribeMessage('fetch-board-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async fetchBoard(socket: Socket, data: FetchBoardMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.boardId, action: 'fetch-board' });
@@ -179,6 +222,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-card-request')
@@ -193,6 +237,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-column-request')
@@ -207,9 +252,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-column-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateColumnTitle(socket: Socket, data: UpdateColumnTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.columnId, action: 'update-column-title' });
@@ -221,6 +268,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-board-visibility-request')
@@ -235,6 +283,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-column-request')
@@ -249,9 +298,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('fetch-card-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async fetchCards(socket: Socket, data: FetchCardsMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.cardIds[0], action: 'fetch-card' });
@@ -264,6 +315,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-element-request')
@@ -282,9 +334,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-element-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateElement(socket: Socket, data: UpdateContentElementMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.elementId, action: 'update-element' });
@@ -296,6 +350,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-element-request')
@@ -310,6 +365,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-element-request')
@@ -324,6 +380,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	private async buildBoardSocketEmitter({ socket, id, action }: { socket: Socket; id: string; action: string }) {