BC-7261 - board websocket metrics

ansible/roles/schulcloud-server-core/tasks/main.yml

@@ -322,3 +322,10 @@
       template: board-collaboration-ingress.yml.j2
       apply: yes
       state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
+
+  - name: BoardCollaborationServiceMonitor
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: '{{ NAMESPACE }}'
+      template: board-collaboration-svc-monitor.yml.j2
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"

ansible/roles/schulcloud-server-core/templates/board-collaboration-deployment.yml.j2

@@ -49,9 +49,9 @@ spec:
         - containerPort: 4450
           name: websocket
           protocol: TCP
-        # - containerPort: 9090
-        #  name: api-metrics
-        #  protocol: TCP
+        - containerPort: 9090
+          name: api-metrics
+          protocol: TCP
         envFrom:
         - configMapRef:
             name: api-configmap

ansible/roles/schulcloud-server-core/templates/board-collaboration-service.yml.j2

@@ -13,9 +13,9 @@ spec:
       targetPort: 4450
       protocol: TCP
       name: websocket
-    #- port: {{ PORT_METRICS_SERVER }}
-    #  targetPort: 9090 # TODO
-    #  protocol: TCP
-    #  name: api-metrics
+    - port: {{ PORT_METRICS_SERVER }}
+      targetPort: 9090 # TODO
+      protocol: TCP
+      name: api-metrics
   selector:
     app: board-collaboration

ansible/roles/schulcloud-server-core/templates/board-collaboration-svc-monitor.yml.j2

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: board-collaboration-svc-monitor
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: board-collaboration
+spec:
+  selector:
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: in
+        values:
+          - board-collaboration-svc
+  endpoints:
+  - path: /metrics
+    port: api-metrics

apps/server/src/apps/board-collaboration.app.ts

@@ -8,44 +8,51 @@ import { install as sourceMapInstall } from 'source-map-support';
 // application imports
 import { SwaggerDocumentOptions } from '@nestjs/swagger';
 import { DB_URL } from '@src/config';
-import { LegacyLogger } from '@src/core/logger';
+import { LegacyLogger, Logger } from '@src/core/logger';
 import { MongoIoAdapter } from '@src/infra/socketio';
 import { BoardCollaborationModule } from '@src/modules/board/board-collaboration.module';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
+import express from 'express';
+import {
+	addPrometheusMetricsMiddlewaresIfEnabled,
+	createAndStartPrometheusMetricsAppIfEnabled,
+} from '@src/apps/helpers/prometheus-metrics';
+import { ExpressAdapter } from '@nestjs/platform-express';
 
 async function bootstrap() {
 	sourceMapInstall();
 
-	const nestApp = await NestFactory.create(BoardCollaborationModule);
-
-	// WinstonLogger
+	const nestExpress = express();
+	const nestExpressAdapter = new ExpressAdapter(nestExpress);
+	const nestApp = await NestFactory.create(BoardCollaborationModule, nestExpressAdapter);
 	nestApp.useLogger(await nestApp.resolve(LegacyLogger));
-
-	// customize nest app settings
 	nestApp.enableCors({ exposedHeaders: ['Content-Disposition'] });
 
 	const mongoIoAdapter = new MongoIoAdapter(nestApp);
 	await mongoIoAdapter.connectToMongoDb(DB_URL);
-
 	nestApp.useWebSocketAdapter(mongoIoAdapter);
 
 	const options: SwaggerDocumentOptions = {
 		operationIdFactory: (_controllerKey: string, methodKey: string) => methodKey,
 	};
 	enableOpenApiDocs(nestApp, 'docs', options);
+	const logger = await nestApp.resolve(Logger);
 
 	await nestApp.init();
 
+	addPrometheusMetricsMiddlewaresIfEnabled(logger, nestExpress);
 	const port = 4450;
 	const basePath = '/board-collaboration';
 
 	nestApp.setGlobalPrefix(basePath);
-	await nestApp.listen(port);
+	await nestApp.listen(port, () => {
+		createAndStartPrometheusMetricsAppIfEnabled(logger);
+	});
 
 	console.log('##########################################');
-	console.log(`### Start Board Collaboration Server   ###`);
-	console.log(`### Port:      ${port}                     ###`);
-	console.log(`### Base path: ${basePath}             ###`);
+	console.log(`### Start Board Collaboration Server`);
+	console.log(`### Port:      ${port}`);
+	console.log(`### Base path: ${basePath}`);
 	console.log('##########################################');
 }
 void bootstrap();

apps/server/src/modules/board/board-ws-api.module.ts

@@ -2,13 +2,15 @@ import { forwardRef, Module } from '@nestjs/common';
 import { CourseRepo } from '@shared/repo';
 import { LoggerModule } from '@src/core/logger';
 import { AuthorizationModule } from '../authorization';
+import { UserModule } from '../user';
 import { BoardModule } from './board.module';
 import { BoardCollaborationGateway } from './gateway/board-collaboration.gateway';
+import { MetricsService } from './metrics/metrics.service';
 import { BoardUc, CardUc, ColumnUc, ElementUc } from './uc';
 
 @Module({
-	imports: [BoardModule, forwardRef(() => AuthorizationModule), LoggerModule],
-	providers: [BoardCollaborationGateway, CardUc, ColumnUc, ElementUc, BoardUc, CourseRepo],
+	imports: [BoardModule, forwardRef(() => AuthorizationModule), LoggerModule, UserModule],
+	providers: [BoardCollaborationGateway, CardUc, ColumnUc, ElementUc, BoardUc, CourseRepo, MetricsService],
 	exports: [],
 })
 export class BoardWsApiModule {}

apps/server/src/modules/board/gateway/board-collaboration.gateway.ts

@@ -1,10 +1,19 @@
 import { WsValidationPipe, Socket } from '@infra/socketio';
 import { MikroORM, UseRequestContext } from '@mikro-orm/core';
 import { UseGuards, UsePipes } from '@nestjs/common';
-import { SubscribeMessage, WebSocketGateway, WsException } from '@nestjs/websockets';
+import {
+	OnGatewayConnection,
+	OnGatewayDisconnect,
+	SubscribeMessage,
+	WebSocketGateway,
+	WebSocketServer,
+	WsException,
+} from '@nestjs/websockets';
 import { WsJwtAuthGuard } from '@src/modules/authentication/guard/ws-jwt-auth.guard';
+import { Server } from 'socket.io';
 import { BoardResponseMapper, CardResponseMapper, ContentElementResponseFactory } from '../controller/mapper';
 import { ColumnResponseMapper } from '../controller/mapper/column-response.mapper';
+import { MetricsService } from '../metrics/metrics.service';
 import { BoardDoAuthorizableService } from '../service';
 import { BoardUc, CardUc, ColumnUc, ElementUc } from '../uc';
 import {
@@ -28,26 +37,65 @@ import { UpdateBoardVisibilityMessageParams } from './dto/update-board-visibilit
 import { UpdateCardHeightMessageParams } from './dto/update-card-height.message.param';
 import { UpdateCardTitleMessageParams } from './dto/update-card-title.message.param';
 import { UpdateContentElementMessageParams } from './dto/update-content-element.message.param';
+import { TrackExecutionTime } from '../metrics/track-execution-time.decorator';
 
 @UsePipes(new WsValidationPipe())
 @WebSocketGateway(BoardCollaborationConfiguration.websocket)
 @UseGuards(WsJwtAuthGuard)
-export class BoardCollaborationGateway {
+export class BoardCollaborationGateway implements OnGatewayConnection, OnGatewayDisconnect {
+	@WebSocketServer()
+	server?: Server;
+
+	// TODO: use loggables instead of legacy logger
 	constructor(
 		private readonly orm: MikroORM,
 		private readonly boardUc: BoardUc,
 		private readonly columnUc: ColumnUc,
 		private readonly cardUc: CardUc,
 		private readonly elementUc: ElementUc,
+		private readonly metricsService: MetricsService,
 		private readonly authorizableService: BoardDoAuthorizableService // to be removed
 	) {}
 
+	trackExecutionTime(methodName: string, executionTimeMs: number) {
+		if (this.metricsService) {
+			this.metricsService.setExecutionTime(methodName, executionTimeMs);
+		}
+	}
+
 	private getCurrentUser(socket: Socket) {
 		const { user } = socket.handshake;
 		if (!user) throw new WsException('Not Authenticated.');
 		return user;
 	}
 
+	private async updateRoomsAndUsersMetrics(socket: Socket) {
+		if (!this.server) {
+			throw new Error('Server is not initialized');
+		}
+
+		const roomCount = Array.from(this.server.of('/').adapter.rooms.keys()).filter((key) =>
+			key.startsWith('board_')
+		).length;
+		this.metricsService.setNumberOfBoardRooms(roomCount);
+		const { user } = socket.handshake;
+		await this.metricsService.trackRoleOfClient(socket.id, user?.userId);
+	}
+
+	public handleConnection(socket: Socket): Promise<void> {
+		if (!socket) {
+			throw new Error('Server is not initialized');
+		}
+		return this.updateRoomsAndUsersMetrics(socket);
+	}
+
+	public handleDisconnect(socket: Socket): void {
+		if (!socket) {
+			throw new Error('Server is not initialized');
+		}
+		this.metricsService.untrackClient(socket.id);
+	}
+
 	@SubscribeMessage('delete-board-request')
 	@UseRequestContext()
 	async deleteBoard(socket: Socket, data: DeleteBoardMessageParams) {
@@ -60,9 +108,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-board-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateBoardTitle(socket: Socket, data: UpdateBoardTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.boardId, action: 'update-board-title' });
@@ -74,9 +124,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-card-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateCardTitle(socket: Socket, data: UpdateCardTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.cardId, action: 'update-card-title' });
@@ -88,6 +140,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-card-height-request')
@@ -102,6 +155,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-card-request')
@@ -116,9 +170,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-card-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async createCard(socket: Socket, data: CreateCardMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.columnId, action: 'create-card' });
@@ -134,6 +190,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-column-request')
@@ -161,6 +218,7 @@ export class BoardCollaborationGateway {
 	}
 
 	@SubscribeMessage('fetch-board-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async fetchBoard(socket: Socket, data: FetchBoardMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.boardId, action: 'fetch-board' });
@@ -173,6 +231,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-card-request')
@@ -187,6 +246,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-column-request')
@@ -201,9 +261,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-column-title-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateColumnTitle(socket: Socket, data: UpdateColumnTitleMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.columnId, action: 'update-column-title' });
@@ -215,6 +277,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-board-visibility-request')
@@ -229,6 +292,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-column-request')
@@ -243,9 +307,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('fetch-card-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async fetchCards(socket: Socket, data: FetchCardsMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.cardIds[0], action: 'fetch-card' });
@@ -258,6 +324,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('create-element-request')
@@ -276,9 +343,11 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('update-element-request')
+	@TrackExecutionTime()
 	@UseRequestContext()
 	async updateElement(socket: Socket, data: UpdateContentElementMessageParams) {
 		const emitter = await this.buildBoardSocketEmitter({ socket, id: data.elementId, action: 'update-element' });
@@ -290,6 +359,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('delete-element-request')
@@ -304,6 +374,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	@SubscribeMessage('move-element-request')
@@ -318,6 +389,7 @@ export class BoardCollaborationGateway {
 		} catch (err) {
 			emitter.emitFailure(data);
 		}
+		await this.updateRoomsAndUsersMetrics(socket);
 	}
 
 	private async buildBoardSocketEmitter({ socket, id, action }: { socket: Socket; id: string; action: string }) {