feat(server): enhance database health checks (#11)

labring · Jul 1, 2024 · d1c2629 · d1c2629
1 parent fc38f74
commit d1c2629
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 4 deletions.
diff --git a/server/src/database/dedicated-database/dedicated-database-task.service.ts b/server/src/database/dedicated-database/dedicated-database-task.service.ts
@@ -6,7 +6,7 @@ import {
   DedicatedDatabaseState,
 } from '../entities/dedicated-database'
 import { DedicatedDatabaseService } from './dedicated-database.service'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { Injectable, Logger } from '@nestjs/common'
 import { RegionService } from 'src/region/region.service'
 import { ClusterService } from 'src/region/cluster/cluster.service'
@@ -25,6 +25,10 @@ export class DedicatedDatabaseTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_INSTANCE_TASK) {
+      return
+    }
+
     this.handleDeletingPhase().catch((err) => {
       this.logger.error(err)
     })
@@ -95,9 +99,12 @@ export class DedicatedDatabaseTaskService {
       return
     }
 
+    const connectionOk = await this.dbService.databaseConnectionIsOk(appid)
+
     manifest = await this.dbService.getDeployManifest(region, user, appid)
     const unavailable = manifest?.status?.phase !== 'Running'
-    if (unavailable) {
+
+    if (unavailable && !connectionOk) {
       await this.relock(appid, waitingTime)
       return
     }

diff --git a/server/src/database/dedicated-database/dedicated-database.service.ts b/server/src/database/dedicated-database/dedicated-database.service.ts
@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common'
+import { Injectable, Logger } from '@nestjs/common'
 import { Region } from 'src/region/entities/region'
 import {
   DedicatedDatabase,
@@ -36,6 +36,8 @@ const p_exec = promisify(exec)
 
 @Injectable()
 export class DedicatedDatabaseService {
+  private readonly logger = new Logger(DedicatedDatabase.name)
+
   constructor(
     private readonly cluster: ClusterService,
     private readonly mongoService: MongoService,
@@ -288,7 +290,7 @@ export class DedicatedDatabaseService {
         `mongodump --uri='${connectionUri}' --gzip --archive=${filePath}`,
       )
     } catch (error) {
-      console.error(`failed to export db ${appid}`, error)
+      this.logger.error(`failed to export db ${appid}`, error)
       throw error
     } finally {
       await SystemDatabase.db
@@ -387,6 +389,40 @@ export class DedicatedDatabaseService {
     }
   }
 
+  async databaseConnectionIsOk(appid: string): Promise<boolean> {
+    try {
+      const { client } = await this.findAndConnect(appid)
+      const admin = client.db('admin')
+      const replSetStatus = await admin.command({ replSetGetStatus: 1 })
+      const members = replSetStatus.members
+      const replicaSetOk: boolean = replSetStatus.ok === 1
+
+      const healthyMembers = members.filter(
+        (member: any) => member.health === 1,
+      )
+
+      const primary = healthyMembers.find(
+        (member: any) => member.stateStr === 'PRIMARY',
+      )
+
+      const majorityCount = Math.ceil(members.length / 2)
+
+      const isClusterHealthy =
+        replicaSetOk && primary && healthyMembers.length >= majorityCount
+
+      if (isClusterHealthy) {
+        return true
+      }
+
+      return false
+    } catch (error) {
+      this.logger.verbose(
+        `dedicatedDatabase health check failed ${appid}\n${error.message}`,
+      )
+      return false
+    }
+  }
+
   async checkDatabaseSyncLimit(uid: ObjectId) {
     const count = await SystemDatabase.db
       .collection<DatabaseSyncRecord>('DatabaseSyncRecord')