From 490de275e079da9df120c662c66c6c508fa0d87a Mon Sep 17 00:00:00 2001 From: williamlardier Date: Fri, 17 May 2024 17:02:36 +0200 Subject: [PATCH 1/3] S3UTILS-163: adapt inflight support We want to compute the diff between the inflight for each bucket at the beginning of its scan with the inflight at the end of the count-items job. this ensure that we do not lose any information between the start of a scan and the end of count-items, that would cause quotas to be exceeded as inflights are lost. The limitation of this implementation comes from the fact that inflights are not created in buckets when there is only quotas on Accounts. This is something we will improve on Scuba:Cloudserver side; so the introduced implementation will be future-proof. --- CountItems/utils/utils.js | 9 ++- utils/S3UtilsMongoClient.js | 129 +++++++++++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/CountItems/utils/utils.js b/CountItems/utils/utils.js index ab957d9c..e4c24e2a 100644 --- a/CountItems/utils/utils.js +++ b/CountItems/utils/utils.js @@ -17,6 +17,7 @@ function consolidateDataMetrics(target, source) { _currentRestoring: 0, _nonCurrentRestored: 0, _nonCurrentRestoring: 0, + _inflightsPreScan: 0, }, }); } @@ -38,7 +39,7 @@ function consolidateDataMetrics(target, source) { if (!source) { return resTarget; } - const { usedCapacity, objectCount } = source; + const { usedCapacity, objectCount, accountOwnerID } = source; resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0; resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0; resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0; @@ -58,7 +59,11 @@ function consolidateDataMetrics(target, source) { resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0; resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0; - // Current and NonCurrent are the total of all other metrics + resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0; + if (accountOwnerID) { + resTarget.accountOwnerID = accountOwnerID; + } + resTarget.usedCapacity.current += usedCapacity ? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring : 0; resTarget.usedCapacity.nonCurrent += usedCapacity diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index ab7b3689..8415a61c 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -74,6 +74,89 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + async updateInflightDeltas(allMetrics, log) { + let cursor; + try { + if (!allMetrics || !Array.isArray(allMetrics) || allMetrics.length === 0) { + return allMetrics; + } + + cursor = await this.getCollection(INFOSTORE).find({}, { + projection: { + 'usedCapacity._inflight': 1, + }, + }); + + const inflights = await cursor.toArray(); + // convert inflights to a map with _id: usedCapacity._inflight + const inflightsMap = inflights.reduce((map, obj) => { + const inflightLong = obj.usedCapacity && obj.usedCapacity._inflight ? obj.usedCapacity._inflight : 0; + return { + ...map, + [obj._id]: inflightLong, + }; + }, {}); + + const accountInflights = {}; + allMetrics.forEach(entry => { + const id = entry._id; + if (id.startsWith('bucket_')) { + const inflightDocument = inflightsMap[id]; + const inflight = Long.fromNumber(Number(inflightDocument ? Math.max(0, inflightDocument - entry.usedCapacity._inflightsPreScan) : 0)); + if (inflight) { + const inflightLong = Long.fromNumber(Number(inflight)); + // Inflights remaining after the scan are part of the "current" bytes, + // and stored in _inflightsDelta + // eslint-disable-next-line no-param-reassign + entry.usedCapacity.current = Long.fromNumber(Number(entry.usedCapacity.current)).add(inflightLong); + // eslint-disable-next-line no-param-reassign + entry.usedCapacity._inflightsDelta = inflightLong; + const accountOwnerId = `account_${entry.accountOwnerID}`; + if (accountInflights[accountOwnerId]) { + accountInflights[accountOwnerId] = Long.fromNumber(Number(accountInflights[accountOwnerId])).add(inflightLong); + } else { + accountInflights[accountOwnerId] = inflightLong; + } + // eslint-disable-next-line no-param-reassign + delete entry.usedCapacity._inflightsPreScan; + // eslint-disable-next-line no-param-reassign + delete entry.accountOwnerID; + } + } + }); + + allMetrics.forEach(entry => { + const id = entry._id; + if (id.startsWith('account_')) { + if (accountInflights[id]) { + // Inflights remaining after the scan are part of the "current" bytes, + // and stored in _inflightsDelta + // eslint-disable-next-line no-param-reassign + entry.usedCapacity.current = Long.fromNumber(Number(entry.usedCapacity.current)).add(accountInflights[id]); + // eslint-disable-next-line no-param-reassign + entry.usedCapacity._inflightsDelta = accountInflights[id]; + } + } + }); + + return allMetrics; + } catch (err) { + log.error('An error occurred', { + method: 'updateInflightDeltas', + errDetails: { ...err }, + errorString: err.toString(), + }); + return allMetrics; + } finally { + if (cursor && !cursor.closed) { + log.info('Finished processing cursor', { + method: 'updateInflightDeltas', + }); + cursor.close(); + } + } + } + async getObjectMDStats(bucketName, bucketInfo, isTransient, log, callback) { let cursor; try { @@ -100,6 +183,9 @@ class S3UtilsMongoClient extends MongoClientInterface { account: {}, // account level metrics }; let stalledCount = 0; + let bucketKey; + let inflightsPreScan = 0; + let accountBucket; const cmpDate = new Date(); cmpDate.setHours(cmpDate.getHours() - 1); @@ -111,6 +197,14 @@ class S3UtilsMongoClient extends MongoClientInterface { return callback(errors.InternalError); } + const bucketEntry = usersBucketCreationDatesMap[`${bucketInfo.getOwner()}${constants.splitter}${bucketName}`]; + if (bucketEntry) { + bucketKey = `bucket_${bucketName}_${new Date(usersBucketCreationDatesMap[bucketEntry]).getTime()}`; + if (bucketKey) { + inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); + } + } + let startCursorDate = new Date(); let processed = 0; await cursor.forEach( @@ -234,6 +328,8 @@ class S3UtilsMongoClient extends MongoClientInterface { collRes.account[account].locations[location].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0; }); }); + // one bucket has only one account + [accountBucket] = Object.keys(collRes.account); monitoring.objectsCount.inc({ status: 'success' }); processed++; }, @@ -261,6 +357,16 @@ class S3UtilsMongoClient extends MongoClientInterface { const retResult = this._handleResults(collRes, isVer); retResult.stalled = stalledCount; + if (inflightsPreScan > 0 && retResult && retResult.dataMetrics) { + Object.keys(retResult.dataMetrics.bucket).forEach(key => { + retResult.dataMetrics.bucket[key].usedCapacity = { + ...retResult.dataMetrics.bucket[key].usedCapacity, + _inflightsPreScan: inflightsPreScan, + }; + retResult.dataMetrics.bucket[key].accountOwnerID = accountBucket; + }); + } + return callback(null, retResult); } catch (err) { log.error('An error occurred', { @@ -654,7 +760,7 @@ class S3UtilsMongoClient extends MongoClientInterface { async updateStorageConsumptionMetrics(countItems, dataMetrics, log, cb) { try { - const updatedStorageMetricsList = [ + let updatedStorageMetricsList = [ { _id: __COUNT_ITEMS, value: countItems }, // iterate every resource through dataMetrics and add to updatedStorageMetricsList ...Object.entries(dataMetrics) @@ -668,6 +774,9 @@ class S3UtilsMongoClient extends MongoClientInterface { ]; log.info('updateStorageConsumptionMetrics: updating storage metrics'); + // update the inflights + updatedStorageMetricsList = await this.updateInflightDeltas(updatedStorageMetricsList, log); + // Drop the temporary collection if it exists try { await this.getCollection(INFOSTORE_TMP).drop(); @@ -711,6 +820,24 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + async readStorageConsumptionInflights(entityName, log) { + try { + const i = this.getCollection(INFOSTORE); + const doc = await i.findOne({ _id: entityName }); + if (!doc || !doc.usedCapacity || !doc.usedCapacity._inflight) { + return 0; + } + return doc.usedCapacity._inflight; + } catch (err) { + log.error('readStorageConsumptionInflights: error reading metrics', { + error: err, + errDetails: { ...err }, + errorString: err.toString(), + }); + return 0; + } + } + /* * Overwrite the getBucketInfos method to specially handle the cases that * bucket collection exists but bucket is not in metastore collection. From 5ae36f1c39c5766e6e86663844a6e8f876b9b319 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Fri, 17 May 2024 17:07:04 +0200 Subject: [PATCH 2/3] S3UTILS-163: adapt and add tests for inflights --- tests/unit/CountItems/CountManager.js | 232 ++++++++++++++ tests/unit/CountItems/utils/utils.js | 60 ++++ tests/unit/utils/S3UtilsMongoClient.js | 411 ++++++++++++++++++++++++- 3 files changed, 689 insertions(+), 14 deletions(-) diff --git a/tests/unit/CountItems/CountManager.js b/tests/unit/CountItems/CountManager.js index d323b0ce..fd8b774f 100644 --- a/tests/unit/CountItems/CountManager.js +++ b/tests/unit/CountItems/CountManager.js @@ -199,6 +199,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -222,6 +223,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -249,6 +251,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -274,6 +277,235 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }); + }); + + test('should update dataMetrics with inflights', () => { + const workers = createWorkers(1); + const m = new CountManager({ + log: new DummyLogger(), + workers, + maxConcurrent: 1, + }); + expect(m.dataMetrics).toEqual({ + account: {}, + bucket: {}, + location: {}, + }); + m._consolidateData({ + dataMetrics: { + account: { + account1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + location1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + bucket1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + location1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }); + expect(m.dataMetrics).toEqual({ + account: { + account1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + location1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + bucket1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + location1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, diff --git a/tests/unit/CountItems/utils/utils.js b/tests/unit/CountItems/utils/utils.js index 812c8e06..12a8e913 100644 --- a/tests/unit/CountItems/utils/utils.js +++ b/tests/unit/CountItems/utils/utils.js @@ -5,6 +5,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 0, nonCurrent: 0, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -29,6 +30,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 10, nonCurrent: 10, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -53,6 +55,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 20, nonCurrent: 20, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -73,6 +76,56 @@ describe('CountItems::utils::consolidateDataMetrics', () => { }, }; + const exampleWithInflights = { + usedCapacity: { + current: 20, + nonCurrent: 20, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + objectCount: { + current: 20, + nonCurrent: 20, + deleteMarker: 20, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }; + + const expectedResponseWithInflights = { + usedCapacity: { + current: 40, + nonCurrent: 40, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + objectCount: { + current: 40, + nonCurrent: 40, + deleteMarker: 40, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }; + test('should return zero-value if target and source are both undefined', () => { const res = consolidateDataMetrics(undefined, undefined); expect(res).toEqual(zeroValueRes); @@ -114,4 +167,11 @@ describe('CountItems::utils::consolidateDataMetrics', () => { const res = consolidateDataMetrics(target, source); expect(res).toEqual(zeroValueRes); }); + + test('should consolidate inflight delta metrics', () => { + const source = exampleWithInflights; + const target = example1; + const res = consolidateDataMetrics(target, source); + expect(res).toEqual(expectedResponseWithInflights); + }); }); diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index 2884623f..f16449fb 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -1,11 +1,13 @@ global.TextEncoder = require('util').TextEncoder; global.TextDecoder = require('util').TextDecoder; +const sinon = require('sinon'); const async = require('async'); const assert = require('assert'); const werelogs = require('werelogs'); +const { Long } = require('mongodb'); const { BucketInfo, ObjectMD, ObjectMDArchive } = require('arsenal').models; const { MongoMemoryReplSet } = require('mongodb-memory-server'); -const { constants } = require('arsenal'); +const { constants, errors } = require('arsenal'); const S3UtilsMongoClient = require('../../../utils/S3UtilsMongoClient'); const { mongoMemoryServerParams, @@ -1003,6 +1005,10 @@ describe('S3UtilsMongoClient, tests', () => { next => repl.stop() .then(() => next()) .catch(next), + next => { + sinon.restore(); + next(); + }, ], done)); const nonVersionedObjectMdTemp = { @@ -2122,12 +2128,227 @@ describe('S3UtilsMongoClient, tests', () => { }, }, ], + [ + 'getObjectMDStats() should return correct results for buckets with inflights', + { + bucketName: 'test-bucket-inflights', + isVersioned: true, + inflights: 1000, + objectList: [ + // versioned object 1, + { + ...objectMdTemp, + versioning: true, + }, + // versioned object 2, + { + ...objectMdTemp, + versioning: true, + }, + // stalled object 1 + { + ...objectMdTemp, + versioning: true, + lastModified: new Date(Date.now() - hr), + repInfo: { + ...objectMdTemp.repInfo, + status: 'PENDING', + backends: [ + { + status: 'PENDING', + site: 'rep-loc-1', + }, + ], + }, + }, + // null versioned object + { + name: 'nullkey', + isNull: true, + ownerId: testAccountCanonicalId, + lastModified: new Date(Date.now() - hr), + }, + ], + }, + { + dataManaged: { + locations: { + 'rep-loc-1': { + curr: 0, + prev: 200, + }, + 'us-east-1': { + curr: 200, + prev: 200, + }, + }, + total: { + curr: 200, + prev: 400, + }, + }, + objects: 2, + stalled: 1, + versions: 2, + dataMetrics: { + account: { + [testAccountCanonicalId]: { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 0, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + 'us-east-1': { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + [`test-bucket-inflights_${testBucketCreationDate}`]: { + accountOwnerID: 'd1d40abd2bd8250962f7f5774af1bbbeaec9b77a0853749d41ec46f142e66fe4', + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + 'rep-loc-1': { + objectCount: { + current: 0, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 0, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + 'us-east-1': { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + ], ]; tests.forEach(([msg, testCase, expected]) => it(msg, done => { const { bucketName, isVersioned, objectList, + inflights, } = testCase; return async.waterfall([ next => createBucket(client, bucketName, isVersioned, err => next(err)), @@ -2144,24 +2365,186 @@ describe('S3UtilsMongoClient, tests', () => { ), next => uploadObjects(client, bucketName, objectList, err => next(err)), next => client.getBucketAttributes(bucketName, logger, next), - (bucketInfo, next) => client.getObjectMDStats( - bucketName, - BucketInfo.fromObj(bucketInfo), - false, - logger, - (err, res) => { - if (err) { - return next(err); - } - assert.deepStrictEqual(res, expected); - return next(); - }, - ), + (bucketInfo, next) => { + if (inflights) { + const mock = sinon.stub(client, 'readStorageConsumptionInflights'); + mock.onFirstCall().returns(Promise.resolve(inflights)); + mock.onSecondCall().returns(Promise.resolve(inflights * 1.5)); + } + return client.getObjectMDStats( + bucketName, + BucketInfo.fromObj(bucketInfo), + false, + logger, + (err, res) => { + if (err) { + return next(err); + } + assert.deepStrictEqual(res, expected); + return next(); + }, + ); + }, next => client.deleteBucket(bucketName, logger, next), ], done); })); }); +describe('S3UtilsMongoClient, update inflight deltas', () => { + let client; + let repl; + + const metrics = [ + { + _id: 'bucket_bucket1_1715849127256', + measuredOn: '2024-05-17T16:08:04.113Z', + accountOwnerID: '1234', + usedCapacity: { + current: 1000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflightsPreScan: 100, + _inflight: 100, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + { + _id: 'bucket_bucket2_1715849127257', + measuredOn: '2024-05-17T16:08:04.113Z', + accountOwnerID: '1234', + usedCapacity: { + current: 1000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflightsPreScan: 1500, + _inflight: 1500, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + { + _id: 'account_1234', + measuredOn: '2024-05-17T16:08:04.113Z', + usedCapacity: { + current: 2000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflight: 0, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + ]; + + beforeAll(async done => { + repl = await MongoMemoryReplSet.create(mongoMemoryServerParams); + client = new S3UtilsMongoClient({ + ...createMongoParamsFromMongoMemoryRepl(repl), + logger, + }); + return client.setup(done); + }); + + afterEach(done => { + sinon.restore(); + done(); + }); + + afterAll(done => async.series([ + next => client.close(next), + next => repl.stop() + .then(() => next()) + .catch(next), + ], done)); + + it('should do nothing if the metrics are empty', async () => { + const input = []; + const output = await client.updateInflightDeltas(input, logger); + assert.strictEqual(input, output); + }); + + it('should return the metrics in case of error', async () => { + sinon.stub(client, 'getCollection').rejects(errors.InternalError); + const output = await client.updateInflightDeltas(metrics, logger); + assert.equal(metrics, output); + }); + + it('should properly compute the inflights deltas', async () => { + sinon.stub(client, 'getCollection').returns({ + find: async () => ({ + toArray: async () => [ + { + _id: 'bucket_bucket1_1715849127256', + usedCapacity: { + _inflight: 3000, + }, + }, + { + _id: 'bucket_bucket2_1715849127257', + usedCapacity: { + _inflight: 5000, + }, + }, + ], + close: async () => {}, + }), + }); + const output = await client.updateInflightDeltas([ + ...metrics, + ], logger); + // first bucket: 1000 current + (3000 post scan - 100 pre scan) = 3900 + assert.strictEqual(output[0].usedCapacity.current.toNumber(), new Long('3900').toNumber()); + // second bucket: 1000 current + (5000 post scan - 1500 pre scan) = 4500 + assert.strictEqual(output[1].usedCapacity.current.toNumber(), new Long('4500').toNumber()); + // for account, we have the current and the sum of bucket's inflight deltas + // as they belong to this account: 2000 + 2900 + 3500 + assert.strictEqual(output[2].usedCapacity.current.toNumber(), new Long('8400').toNumber()); + }); +}); + describe('S3UtilsMongoClient, cold object helpers', () => { const coldObjectMdTemp = { name: 'coldkey', From 5eca97582086a71ddec256ed466bf891301e7a1a Mon Sep 17 00:00:00 2001 From: williamlardier Date: Fri, 17 May 2024 17:13:14 +0200 Subject: [PATCH 3/3] S3UTILS-163: bump package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5e92136b..c8d46f72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.14.7", + "version": "1.14.8", "engines": { "node": ">= 16" },