From 54dcc413949dfe2099966d32d342cd5236bffe87 Mon Sep 17 00:00:00 2001 From: Thomas Kortyka Date: Fri, 8 Nov 2024 00:31:20 +0100 Subject: [PATCH] chore: add kafka --- .../src/app/health/health.controller.ts | 20 ++++ .../src/app/health/health.module.ts | 24 ++++ package-lock.json | 113 ++++++++++++------ package.json | 4 +- 4 files changed, 122 insertions(+), 39 deletions(-) diff --git a/apps/azkaban-gateway/src/app/health/health.controller.ts b/apps/azkaban-gateway/src/app/health/health.controller.ts index 3c563b74..b5eba57b 100644 --- a/apps/azkaban-gateway/src/app/health/health.controller.ts +++ b/apps/azkaban-gateway/src/app/health/health.controller.ts @@ -17,12 +17,18 @@ export class HealthController { @Inject('MEMORY_RSS_TRESHOLD') private readonly rssTreshold: number, @Inject('BROKER_CONNECTION_STRING') private readonly brokerConnectionString: string, + @Inject('KAFKA_CONNECTION_STRING') + private readonly kafkaConnectionString: string, @Inject('REDIS_HOST_STRING') private readonly redisHostString: string, @Inject('REDIS_PORT_NUMBER') private readonly redisPortNumber: number, @Inject('REDIS_PASSWORD_STRING') private readonly redisPasswordString: string, + @Inject('KAFKA_USER_STRING') + private readonly kafkaUserString: string, + @Inject('KAFKA_PASSWORD_STRING') + private readonly kafkaPasswordString: string, private readonly service: HealthCheckService, private readonly memory: MemoryHealthIndicator, private readonly microservices: MicroserviceHealthIndicator, @@ -44,6 +50,20 @@ export class HealthController { consumerTag: 'gateway-health', }, }), + () => + this.microservices.pingCheck('kafka', { + transport: Transport.KAFKA, + options: { + client: { + brokers: [this.kafkaConnectionString], + sasl: { + mechanism: 'scram-sha-256', + username: this.kafkaUserString, + password: this.kafkaPasswordString, + }, + }, + }, + }), () => this.microservices.pingCheck('redis', { transport: Transport.REDIS, diff --git a/apps/azkaban-gateway/src/app/health/health.module.ts b/apps/azkaban-gateway/src/app/health/health.module.ts index 3965ebad..d43aa187 100644 --- a/apps/azkaban-gateway/src/app/health/health.module.ts +++ b/apps/azkaban-gateway/src/app/health/health.module.ts @@ -38,6 +38,16 @@ import { azkaban_vhost } from '@toxictoast/azkaban-broker-rabbitmq'; }, inject: [ConfigService], }, + { + provide: 'KAFKA_CONNECTION_STRING', + useFactory: (config: ConfigService) => { + const hostname = config.get('KAFKA_HOST', 'localhost'); + const port = config.get('KAFKA_PORT', 9092); + // + return `${hostname}:${port}`; + }, + inject: [ConfigService], + }, { provide: 'REDIS_HOST_STRING', useFactory: (config: ConfigService) => { @@ -59,6 +69,20 @@ import { azkaban_vhost } from '@toxictoast/azkaban-broker-rabbitmq'; }, inject: [ConfigService], }, + { + provide: 'KAFKA_USER_STRING', + useFactory: (config: ConfigService) => { + return config.get('KAFKA_USERNAME', 'guest'); + }, + inject: [ConfigService], + }, + { + provide: 'KAFKA_PASSWORD_STRING', + useFactory: (config: ConfigService) => { + return config.get('KAFKA_PASSWORD', 'supersecret'); + }, + inject: [ConfigService], + }, ], }) export class HealthModule {} diff --git a/package-lock.json b/package-lock.json index 65de403c..2d8c6fec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,8 +34,9 @@ "@reduxjs/toolkit": "^2.2.7", "@tailwindcss/forms": "^0.5.8", "@toxictoast/azkaban-base-domain": "^0.0.2", - "@toxictoast/azkaban-base-helpers": "^0.0.15", + "@toxictoast/azkaban-base-helpers": "^0.0.21", "@toxictoast/azkaban-base-types": "^0.0.1", + "@toxictoast/azkaban-broker-kafka": "^0.0.8", "@toxictoast/azkaban-broker-rabbitmq": "^0.0.45", "@toxictoast/azkaban-sdk": "^0.0.16", "@toxictoast/azkaban-twitch-bot": "^0.0.1", @@ -129,6 +130,7 @@ "jest-environment-jsdom": "29.7.0", "jest-environment-node": "^29.7.0", "jsdom": "~25.0.0", + "kafkajs": "^2.2.4", "nx": "20.0.6", "postcss": "8.4.44", "prettier": "^3.3.3", @@ -4543,9 +4545,9 @@ } }, "node_modules/@nestjs/microservices": { - "version": "10.4.6", - "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.4.6.tgz", - "integrity": "sha512-8mIWmgXvaQeIaOOq6IHVqluGfRKTE9cLfx7HPgOAoGrpNcejZIPnR78u/tHQC8zHmoo4JYJxaDS05kqrCo6Qhw==", + "version": "10.4.7", + "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.4.7.tgz", + "integrity": "sha512-xqwFqheyo5c52s8YFf/vZD6felccoB+FpCCTj66jLxKRk3ZITjiENwar0mWjkiiAz9YoTmsDP80bnEoIpgBLWg==", "license": "MIT", "dependencies": { "iterare": "1.2.1", @@ -7856,29 +7858,45 @@ } }, "node_modules/@toxictoast/azkaban-base-helpers": { - "version": "0.0.15", - "resolved": "https://registry.npmjs.org/@toxictoast/azkaban-base-helpers/-/azkaban-base-helpers-0.0.15.tgz", - "integrity": "sha512-sOq7LW2XRBnJ5Y5WX0+QA8BQp4hr15j2jpMMEhmkMLDcLS3nA6DAqi4uFZj3RVDdVyJoh56bkEUbbhZBoCBbGA==", + "version": "0.0.21", + "resolved": "https://registry.npmjs.org/@toxictoast/azkaban-base-helpers/-/azkaban-base-helpers-0.0.21.tgz", + "integrity": "sha512-x2WfXvAxQxDMdPPytgdxXKYaH6iGAfGvLiQYq3mwYRCd8CencXsA/XaX+UiovXcML+THaEc5iWPqntXLBlb/gg==", "license": "ISC", "dependencies": { + "@nestjs/common": "^10.4.6", + "@nestjs/core": "^10.4.6", + "@nestjs/microservices": "^10.4.7", + "@nestjs/swagger": "^8.0.0", "apialerts-js": "^1.0.1", "bcrypt": "^5.1.1", + "compression": "^1.7.5", + "cookie-parser": "^1.4.7", + "helmet": "^8.0.0", "slugify": "^1.6.6", "typeorm": "^0.3.20", - "uuid": "^10.0.0" + "uuid": "^11.0.0" + } + }, + "node_modules/@toxictoast/azkaban-base-helpers/node_modules/helmet": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-8.0.0.tgz", + "integrity": "sha512-VyusHLEIIO5mjQPUI1wpOAEu+wl6Q0998jzTxqUYGE45xCIcAxy3MsbEK/yyJUJ3ADeMoB6MornPH6GMWAf+Pw==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" } }, "node_modules/@toxictoast/azkaban-base-helpers/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.2.tgz", + "integrity": "sha512-14FfcOJmqdjbBPdDjFQyk/SdT4NySW4eM0zcG+HqbHP5jzuH56xO3J1DGhgs/cEMCfwYi3HQI1gnTO62iaG+tQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], "license": "MIT", "bin": { - "uuid": "dist/bin/uuid" + "uuid": "dist/esm/bin/uuid" } }, "node_modules/@toxictoast/azkaban-base-types": { @@ -7886,6 +7904,17 @@ "resolved": "https://registry.npmjs.org/@toxictoast/azkaban-base-types/-/azkaban-base-types-0.0.1.tgz", "integrity": "sha512-PkXYD4ldTtps2WElLiHVFNIR7dsWlzeGWGsFa3I8BZenhLYJ0q8FI/F9oa0XXHPF3xWtF+XgjZDMKRm9ibwBMw==" }, + "node_modules/@toxictoast/azkaban-broker-kafka": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@toxictoast/azkaban-broker-kafka/-/azkaban-broker-kafka-0.0.8.tgz", + "integrity": "sha512-LOmz5AnlRXIL1qrpfF1mTOOijJtdBxG85oLFip2VgGS0QkqTsTLg/CF4XEMymWurYO8FPmB2TLiZLsmNp5gi9Q==", + "license": "ISC", + "dependencies": { + "@nestjs/common": "^10.3.8", + "@nestjs/microservices": "^10.3.8", + "kafkajs": "^2.2.4" + } + }, "node_modules/@toxictoast/azkaban-broker-rabbitmq": { "version": "0.0.45", "resolved": "https://registry.npmjs.org/@toxictoast/azkaban-broker-rabbitmq/-/azkaban-broker-rabbitmq-0.0.45.tgz", @@ -11869,30 +11898,23 @@ } }, "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==", + "license": "MIT", "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", + "bytes": "3.1.2", + "compressible": "~2.0.18", "debug": "2.6.9", + "negotiator": "~0.6.4", "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", + "safe-buffer": "5.2.1", "vary": "~1.1.2" }, "engines": { "node": ">= 0.8.0" } }, - "node_modules/compression/node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/compression/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -11906,10 +11928,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } }, "node_modules/computeds": { "version": "0.0.1", @@ -12025,11 +12051,12 @@ } }, "node_modules/cookie-parser": { - "version": "1.4.6", - "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.6.tgz", - "integrity": "sha512-z3IzaNjdwUC2olLIB5/ITd0/setiaFMLYiZJle7xg5Fe9KWAceil7xszYfHHBtDFYLSgJduS2Ty0P1uJdPDJeA==", + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz", + "integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==", + "license": "MIT", "dependencies": { - "cookie": "0.4.1", + "cookie": "0.7.2", "cookie-signature": "1.0.6" }, "engines": { @@ -12037,9 +12064,10 @@ } }, "node_modules/cookie-parser/node_modules/cookie": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", - "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -18714,6 +18742,15 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/kafkajs": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", + "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", diff --git a/package.json b/package.json index 8381ad39..88678546 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "jest-environment-jsdom": "29.7.0", "jest-environment-node": "^29.7.0", "jsdom": "~25.0.0", + "kafkajs": "^2.2.4", "nx": "20.0.6", "postcss": "8.4.44", "prettier": "^3.3.3", @@ -104,8 +105,9 @@ "@reduxjs/toolkit": "^2.2.7", "@tailwindcss/forms": "^0.5.8", "@toxictoast/azkaban-base-domain": "^0.0.2", - "@toxictoast/azkaban-base-helpers": "^0.0.15", + "@toxictoast/azkaban-base-helpers": "^0.0.21", "@toxictoast/azkaban-base-types": "^0.0.1", + "@toxictoast/azkaban-broker-kafka": "^0.0.8", "@toxictoast/azkaban-broker-rabbitmq": "^0.0.45", "@toxictoast/azkaban-sdk": "^0.0.16", "@toxictoast/azkaban-twitch-bot": "^0.0.1",