From c081558c54f0a51e5f10455e31e66a5841ba5f7e Mon Sep 17 00:00:00 2001
From: bramjanssen <bram.janssen@vito.be>
Date: Mon, 2 Dec 2024 11:37:45 +0100
Subject: [PATCH] fix: enable dynamic setting of es scroll timeout

---
 src/config/schema/config.schema.ts             |  7 +++++++
 src/jobs/services/database/database.service.ts | 12 ++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/config/schema/config.schema.ts b/src/config/schema/config.schema.ts
index e0a0ae1..c95812c 100644
--- a/src/config/schema/config.schema.ts
+++ b/src/config/schema/config.schema.ts
@@ -76,5 +76,12 @@ export const schema: convict.Schema<AppConfig> = {
       env: 'DB_JOBS_IDX',
       arg: 'db_jobs_idx',
     },
+    jobsScrollTimout: {
+      doc: 'Timeout of the scroll to use in elasticsearch',
+      format: '*',
+      default: '5s',
+      env: 'DB_JOBS_SCOLL_TIMEOUT',
+      arg: 'db_jobs_scroll_timeout',
+    },
   },
 };
diff --git a/src/jobs/services/database/database.service.ts b/src/jobs/services/database/database.service.ts
index 877467b..362347b 100644
--- a/src/jobs/services/database/database.service.ts
+++ b/src/jobs/services/database/database.service.ts
@@ -8,6 +8,7 @@ import { ApiResponse } from '@elastic/elasticsearch';
 @Injectable()
 export class DatabaseService {
   private JOBS_INDEX = '';
+  private JOBS_SCROLL_TIMEOUT = '5s';
 
   constructor(
     private configService: ConfigService,
@@ -15,6 +16,13 @@ export class DatabaseService {
     private logger: Logger,
   ) {
     this.JOBS_INDEX = this.configService.get('database.jobsIdx');
+    this.JOBS_SCROLL_TIMEOUT = this.configService.get(
+      'database.jobsScrollTimout',
+    );
+
+    this.logger.debug(
+      `Starting up databse service for jobs at index ${this.JOBS_INDEX} and scroll timeout of ${this.JOBS_SCROLL_TIMEOUT}`,
+    );
   }
 
   /**
@@ -115,7 +123,7 @@ export class DatabaseService {
     // Perform initial search
     const results = await this.elasticSearch.search({
       index: this.JOBS_INDEX,
-      scroll: '5s',
+      scroll: this.JOBS_SCROLL_TIMEOUT,
       body: query,
       size: limit || 1000,
     });
@@ -134,7 +142,7 @@ export class DatabaseService {
       queue.push(
         await this.elasticSearch.scroll({
           scroll_id: body._scroll_id,
-          scroll: '5s',
+          scroll: this.JOBS_SCROLL_TIMEOUT,
         }),
       );
     }