From 44f59b0aa7b6acfe856d8d43e2188d655c4c4904 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Thu, 19 Sep 2024 14:20:09 -0700 Subject: [PATCH 1/2] bf: S3UTILS-180 unbounded number of sproxyd keys in Map - Fix missing transmission of the environment variable that limits the number of keys kept in the in-memory maps (`OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE`) - Add `Number.parseInt()` calls to environment variables where needed - Sanitize a little bit the `env.js`: - reorder lexicographically - set default value of `OBJECT_REPAIR_RAFT_LOG_BEGIN_CSEQ` to `null` to make ObjectRepair use the computed default when fetched (instead of having it declared separately from `env.js`) Manually tested with a mocked oplog server returning entries with sproxyd keys: - without the fix: memory usage grows quickly (multiple GB after a few minutes) - with the fix: memory stabilizes at a level proportional to the value of `OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE` - not specifying `OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE` in the environment: stabilizes memory usage around the default value of 100000 entries. --- ObjectRepair/SproxydKeysSubscribers.js | 2 +- ObjectRepair/env.js | 20 ++++++++++++++------ ObjectRepair/index.js | 12 +++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/ObjectRepair/SproxydKeysSubscribers.js b/ObjectRepair/SproxydKeysSubscribers.js index 75110341..7c67f5ec 100644 --- a/ObjectRepair/SproxydKeysSubscribers.js +++ b/ObjectRepair/SproxydKeysSubscribers.js @@ -19,7 +19,7 @@ class DuplicateSproxydKeyFoundHandler { this._getObjectURL = getObjectURL; this.queue = queue(this._repairObject, 1); // use OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE since there will be at least one sproxyd key per object - this.visitedObjects = new BoundedMap(env.DUPLICATE_KEYS_WINDOW_SIZE); + this.visitedObjects = new BoundedMap(env.OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE); } /** diff --git a/ObjectRepair/env.js b/ObjectRepair/env.js index 45ed71e2..d5040aec 100644 --- a/ObjectRepair/env.js +++ b/ObjectRepair/env.js @@ -1,12 +1,20 @@ const env = { OBJECT_REPAIR_BUCKETD_HOSTPORT: process.env.OBJECT_REPAIR_BUCKETD_HOSTPORT, - OBJECT_REPAIR_SPROXYD_HOSTPORT: process.env.OBJECT_REPAIR_SPROXYD_HOSTPORT, - OBJECT_REPAIR_RAFT_SESSION_ID: process.env.OBJECT_REPAIR_RAFT_SESSION_ID, - OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE: process.env.OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE || 1000, - OBJECT_REPAIR_LOOKBACK_WINDOW: process.env.OBJECT_REPAIR_LOOKBACK_WINDOW || 10000, - OBJECT_REPAIR_LOG_LEVEL: process.env.OBJECT_REPAIR_LOG_LEVEL || 'info', OBJECT_REPAIR_DUMP_LEVEL: process.env.OBJECT_REPAIR_DUMP_LEVEL || 'error', - OBJECT_REPAIR_LOG_INTERVAL: process.env.OBJECT_REPAIR_LOG_INTERVAL || 300, + OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE: + Number.parseInt(process.env.OBJECT_REPAIR_DUPLICATE_KEYS_WINDOW_SIZE, 10) || 100000, + OBJECT_REPAIR_LOG_INTERVAL: + Number.parseInt(process.env.OBJECT_REPAIR_LOG_INTERVAL, 10) || 300, + OBJECT_REPAIR_LOG_LEVEL: process.env.OBJECT_REPAIR_LOG_LEVEL || 'info', + OBJECT_REPAIR_LOOKBACK_WINDOW: + Number.parseInt(process.env.OBJECT_REPAIR_LOOKBACK_WINDOW, 10) || 10000, + OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE: + Number.parseInt(process.env.OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE, 10) || 1000, + OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ: + Number.parseInt(process.env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ, 10) || null, + OBJECT_REPAIR_RAFT_SESSION_ID: + Number.parseInt(process.env.OBJECT_REPAIR_RAFT_SESSION_ID, 10) || undefined, + OBJECT_REPAIR_SPROXYD_HOSTPORT: process.env.OBJECT_REPAIR_SPROXYD_HOSTPORT, }; module.exports = { env }; diff --git a/ObjectRepair/index.js b/ObjectRepair/index.js index b0ebb07f..d0254bfa 100644 --- a/ObjectRepair/index.js +++ b/ObjectRepair/index.js @@ -44,28 +44,26 @@ Optional environment variables: OBJECT_REPAIR_TLS_CA_PATH: path to TLS alternate CA file `; for (const [key, value] of Object.entries(env)) { - if (!value) { + if (value === undefined) { log.info(`${key} must be defined`); console.error(USAGE); process.exit(1); } } -env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ = process.env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ; - /** * Creates new reader and runs until stop(). * @returns {undefined} */ function runJournalReader() { - if (env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ === undefined) { + if (env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ === null) { log.info('OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ is not defined.' + 'Ingestion will start at latest cseq - OBJECT_REPAIR_LOOKBACK_WINDOW'); } const reader = new RaftJournalReader( - Number.parseInt(env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ, 10), - Number.parseInt(env.OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE, 10), - Number.parseInt(env.OBJECT_REPAIR_RAFT_SESSION_ID, 10), + env.OBJECT_REPAIR_RAFT_LOG_BEGIN_SEQ, + env.OBJECT_REPAIR_RAFT_LOG_BATCH_SIZE, + env.OBJECT_REPAIR_RAFT_SESSION_ID, ); reader.run(); } From 15c504ceaf8490b83b833e02c6c53984ee10a945 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Mon, 23 Sep 2024 10:34:35 -0700 Subject: [PATCH 2/2] S3UTILS-180: bump s3utils version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1ca25934..82518d37 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.14.13", + "version": "1.14.14", "engines": { "node": ">= 16" },