diff --git a/CountItems/utils/utils.js b/CountItems/utils/utils.js index ab957d9c..e4c24e2a 100644 --- a/CountItems/utils/utils.js +++ b/CountItems/utils/utils.js @@ -17,6 +17,7 @@ function consolidateDataMetrics(target, source) { _currentRestoring: 0, _nonCurrentRestored: 0, _nonCurrentRestoring: 0, + _inflightsPreScan: 0, }, }); } @@ -38,7 +39,7 @@ function consolidateDataMetrics(target, source) { if (!source) { return resTarget; } - const { usedCapacity, objectCount } = source; + const { usedCapacity, objectCount, accountOwnerID } = source; resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0; resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0; resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0; @@ -58,7 +59,11 @@ function consolidateDataMetrics(target, source) { resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0; resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0; - // Current and NonCurrent are the total of all other metrics + resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0; + if (accountOwnerID) { + resTarget.accountOwnerID = accountOwnerID; + } + resTarget.usedCapacity.current += usedCapacity ? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring : 0; resTarget.usedCapacity.nonCurrent += usedCapacity diff --git a/package.json b/package.json index 5e92136b..c8d46f72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.14.7", + "version": "1.14.8", "engines": { "node": ">= 16" }, diff --git a/tests/unit/CountItems/CountManager.js b/tests/unit/CountItems/CountManager.js index d323b0ce..fd8b774f 100644 --- a/tests/unit/CountItems/CountManager.js +++ b/tests/unit/CountItems/CountManager.js @@ -199,6 +199,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -222,6 +223,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -249,6 +251,7 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, @@ -274,6 +277,235 @@ describe('CountItems::CountManager', () => { usedCapacity: { current: 200, nonCurrent: 100, + _inflightsPreScan: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }); + }); + + test('should update dataMetrics with inflights', () => { + const workers = createWorkers(1); + const m = new CountManager({ + log: new DummyLogger(), + workers, + maxConcurrent: 1, + }); + expect(m.dataMetrics).toEqual({ + account: {}, + bucket: {}, + location: {}, + }); + m._consolidateData({ + dataMetrics: { + account: { + account1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + location1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + bucket1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + location1: { + objectCount: { + current: 10, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 100, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }); + expect(m.dataMetrics).toEqual({ + account: { + account1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + location1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + bucket1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 100, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + location1: { + objectCount: { + current: 11, + deleteMarker: 0, + nonCurrent: 10, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 1, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 100, + _inflightsPreScan: 1000, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 100, diff --git a/tests/unit/CountItems/utils/utils.js b/tests/unit/CountItems/utils/utils.js index 812c8e06..12a8e913 100644 --- a/tests/unit/CountItems/utils/utils.js +++ b/tests/unit/CountItems/utils/utils.js @@ -5,6 +5,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 0, nonCurrent: 0, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -29,6 +30,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 10, nonCurrent: 10, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -53,6 +55,7 @@ describe('CountItems::utils::consolidateDataMetrics', () => { usedCapacity: { current: 20, nonCurrent: 20, + _inflightsPreScan: 0, _currentCold: 0, _nonCurrentCold: 0, _currentRestored: 0, @@ -73,6 +76,56 @@ describe('CountItems::utils::consolidateDataMetrics', () => { }, }; + const exampleWithInflights = { + usedCapacity: { + current: 20, + nonCurrent: 20, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + objectCount: { + current: 20, + nonCurrent: 20, + deleteMarker: 20, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }; + + const expectedResponseWithInflights = { + usedCapacity: { + current: 40, + nonCurrent: 40, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + objectCount: { + current: 40, + nonCurrent: 40, + deleteMarker: 40, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }; + test('should return zero-value if target and source are both undefined', () => { const res = consolidateDataMetrics(undefined, undefined); expect(res).toEqual(zeroValueRes); @@ -114,4 +167,11 @@ describe('CountItems::utils::consolidateDataMetrics', () => { const res = consolidateDataMetrics(target, source); expect(res).toEqual(zeroValueRes); }); + + test('should consolidate inflight delta metrics', () => { + const source = exampleWithInflights; + const target = example1; + const res = consolidateDataMetrics(target, source); + expect(res).toEqual(expectedResponseWithInflights); + }); }); diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index 2884623f..f16449fb 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -1,11 +1,13 @@ global.TextEncoder = require('util').TextEncoder; global.TextDecoder = require('util').TextDecoder; +const sinon = require('sinon'); const async = require('async'); const assert = require('assert'); const werelogs = require('werelogs'); +const { Long } = require('mongodb'); const { BucketInfo, ObjectMD, ObjectMDArchive } = require('arsenal').models; const { MongoMemoryReplSet } = require('mongodb-memory-server'); -const { constants } = require('arsenal'); +const { constants, errors } = require('arsenal'); const S3UtilsMongoClient = require('../../../utils/S3UtilsMongoClient'); const { mongoMemoryServerParams, @@ -1003,6 +1005,10 @@ describe('S3UtilsMongoClient, tests', () => { next => repl.stop() .then(() => next()) .catch(next), + next => { + sinon.restore(); + next(); + }, ], done)); const nonVersionedObjectMdTemp = { @@ -2122,12 +2128,227 @@ describe('S3UtilsMongoClient, tests', () => { }, }, ], + [ + 'getObjectMDStats() should return correct results for buckets with inflights', + { + bucketName: 'test-bucket-inflights', + isVersioned: true, + inflights: 1000, + objectList: [ + // versioned object 1, + { + ...objectMdTemp, + versioning: true, + }, + // versioned object 2, + { + ...objectMdTemp, + versioning: true, + }, + // stalled object 1 + { + ...objectMdTemp, + versioning: true, + lastModified: new Date(Date.now() - hr), + repInfo: { + ...objectMdTemp.repInfo, + status: 'PENDING', + backends: [ + { + status: 'PENDING', + site: 'rep-loc-1', + }, + ], + }, + }, + // null versioned object + { + name: 'nullkey', + isNull: true, + ownerId: testAccountCanonicalId, + lastModified: new Date(Date.now() - hr), + }, + ], + }, + { + dataManaged: { + locations: { + 'rep-loc-1': { + curr: 0, + prev: 200, + }, + 'us-east-1': { + curr: 200, + prev: 200, + }, + }, + total: { + curr: 200, + prev: 400, + }, + }, + objects: 2, + stalled: 1, + versions: 2, + dataMetrics: { + account: { + [testAccountCanonicalId]: { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 0, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + 'us-east-1': { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + bucket: { + [`test-bucket-inflights_${testBucketCreationDate}`]: { + accountOwnerID: 'd1d40abd2bd8250962f7f5774af1bbbeaec9b77a0853749d41ec46f142e66fe4', + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _inflightsPreScan: 1000, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + location: { + 'rep-loc-1': { + objectCount: { + current: 0, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 0, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + 'us-east-1': { + objectCount: { + current: 2, + deleteMarker: 0, + nonCurrent: 2, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + usedCapacity: { + current: 200, + nonCurrent: 200, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + }, + }, + }, + }, + }, + ], ]; tests.forEach(([msg, testCase, expected]) => it(msg, done => { const { bucketName, isVersioned, objectList, + inflights, } = testCase; return async.waterfall([ next => createBucket(client, bucketName, isVersioned, err => next(err)), @@ -2144,24 +2365,186 @@ describe('S3UtilsMongoClient, tests', () => { ), next => uploadObjects(client, bucketName, objectList, err => next(err)), next => client.getBucketAttributes(bucketName, logger, next), - (bucketInfo, next) => client.getObjectMDStats( - bucketName, - BucketInfo.fromObj(bucketInfo), - false, - logger, - (err, res) => { - if (err) { - return next(err); - } - assert.deepStrictEqual(res, expected); - return next(); - }, - ), + (bucketInfo, next) => { + if (inflights) { + const mock = sinon.stub(client, 'readStorageConsumptionInflights'); + mock.onFirstCall().returns(Promise.resolve(inflights)); + mock.onSecondCall().returns(Promise.resolve(inflights * 1.5)); + } + return client.getObjectMDStats( + bucketName, + BucketInfo.fromObj(bucketInfo), + false, + logger, + (err, res) => { + if (err) { + return next(err); + } + assert.deepStrictEqual(res, expected); + return next(); + }, + ); + }, next => client.deleteBucket(bucketName, logger, next), ], done); })); }); +describe('S3UtilsMongoClient, update inflight deltas', () => { + let client; + let repl; + + const metrics = [ + { + _id: 'bucket_bucket1_1715849127256', + measuredOn: '2024-05-17T16:08:04.113Z', + accountOwnerID: '1234', + usedCapacity: { + current: 1000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflightsPreScan: 100, + _inflight: 100, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + { + _id: 'bucket_bucket2_1715849127257', + measuredOn: '2024-05-17T16:08:04.113Z', + accountOwnerID: '1234', + usedCapacity: { + current: 1000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflightsPreScan: 1500, + _inflight: 1500, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + { + _id: 'account_1234', + measuredOn: '2024-05-17T16:08:04.113Z', + usedCapacity: { + current: 2000, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + _inflight: 0, + }, + objectCount: { + current: 10, + nonCurrent: 0, + _currentCold: 0, + _nonCurrentCold: 0, + _currentRestored: 0, + _currentRestoring: 0, + _nonCurrentRestored: 0, + _nonCurrentRestoring: 0, + deleteMarker: 0, + }, + }, + ]; + + beforeAll(async done => { + repl = await MongoMemoryReplSet.create(mongoMemoryServerParams); + client = new S3UtilsMongoClient({ + ...createMongoParamsFromMongoMemoryRepl(repl), + logger, + }); + return client.setup(done); + }); + + afterEach(done => { + sinon.restore(); + done(); + }); + + afterAll(done => async.series([ + next => client.close(next), + next => repl.stop() + .then(() => next()) + .catch(next), + ], done)); + + it('should do nothing if the metrics are empty', async () => { + const input = []; + const output = await client.updateInflightDeltas(input, logger); + assert.strictEqual(input, output); + }); + + it('should return the metrics in case of error', async () => { + sinon.stub(client, 'getCollection').rejects(errors.InternalError); + const output = await client.updateInflightDeltas(metrics, logger); + assert.equal(metrics, output); + }); + + it('should properly compute the inflights deltas', async () => { + sinon.stub(client, 'getCollection').returns({ + find: async () => ({ + toArray: async () => [ + { + _id: 'bucket_bucket1_1715849127256', + usedCapacity: { + _inflight: 3000, + }, + }, + { + _id: 'bucket_bucket2_1715849127257', + usedCapacity: { + _inflight: 5000, + }, + }, + ], + close: async () => {}, + }), + }); + const output = await client.updateInflightDeltas([ + ...metrics, + ], logger); + // first bucket: 1000 current + (3000 post scan - 100 pre scan) = 3900 + assert.strictEqual(output[0].usedCapacity.current.toNumber(), new Long('3900').toNumber()); + // second bucket: 1000 current + (5000 post scan - 1500 pre scan) = 4500 + assert.strictEqual(output[1].usedCapacity.current.toNumber(), new Long('4500').toNumber()); + // for account, we have the current and the sum of bucket's inflight deltas + // as they belong to this account: 2000 + 2900 + 3500 + assert.strictEqual(output[2].usedCapacity.current.toNumber(), new Long('8400').toNumber()); + }); +}); + describe('S3UtilsMongoClient, cold object helpers', () => { const coldObjectMdTemp = { name: 'coldkey', diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index ab7b3689..8415a61c 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -74,6 +74,89 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + async updateInflightDeltas(allMetrics, log) { + let cursor; + try { + if (!allMetrics || !Array.isArray(allMetrics) || allMetrics.length === 0) { + return allMetrics; + } + + cursor = await this.getCollection(INFOSTORE).find({}, { + projection: { + 'usedCapacity._inflight': 1, + }, + }); + + const inflights = await cursor.toArray(); + // convert inflights to a map with _id: usedCapacity._inflight + const inflightsMap = inflights.reduce((map, obj) => { + const inflightLong = obj.usedCapacity && obj.usedCapacity._inflight ? obj.usedCapacity._inflight : 0; + return { + ...map, + [obj._id]: inflightLong, + }; + }, {}); + + const accountInflights = {}; + allMetrics.forEach(entry => { + const id = entry._id; + if (id.startsWith('bucket_')) { + const inflightDocument = inflightsMap[id]; + const inflight = Long.fromNumber(Number(inflightDocument ? Math.max(0, inflightDocument - entry.usedCapacity._inflightsPreScan) : 0)); + if (inflight) { + const inflightLong = Long.fromNumber(Number(inflight)); + // Inflights remaining after the scan are part of the "current" bytes, + // and stored in _inflightsDelta + // eslint-disable-next-line no-param-reassign + entry.usedCapacity.current = Long.fromNumber(Number(entry.usedCapacity.current)).add(inflightLong); + // eslint-disable-next-line no-param-reassign + entry.usedCapacity._inflightsDelta = inflightLong; + const accountOwnerId = `account_${entry.accountOwnerID}`; + if (accountInflights[accountOwnerId]) { + accountInflights[accountOwnerId] = Long.fromNumber(Number(accountInflights[accountOwnerId])).add(inflightLong); + } else { + accountInflights[accountOwnerId] = inflightLong; + } + // eslint-disable-next-line no-param-reassign + delete entry.usedCapacity._inflightsPreScan; + // eslint-disable-next-line no-param-reassign + delete entry.accountOwnerID; + } + } + }); + + allMetrics.forEach(entry => { + const id = entry._id; + if (id.startsWith('account_')) { + if (accountInflights[id]) { + // Inflights remaining after the scan are part of the "current" bytes, + // and stored in _inflightsDelta + // eslint-disable-next-line no-param-reassign + entry.usedCapacity.current = Long.fromNumber(Number(entry.usedCapacity.current)).add(accountInflights[id]); + // eslint-disable-next-line no-param-reassign + entry.usedCapacity._inflightsDelta = accountInflights[id]; + } + } + }); + + return allMetrics; + } catch (err) { + log.error('An error occurred', { + method: 'updateInflightDeltas', + errDetails: { ...err }, + errorString: err.toString(), + }); + return allMetrics; + } finally { + if (cursor && !cursor.closed) { + log.info('Finished processing cursor', { + method: 'updateInflightDeltas', + }); + cursor.close(); + } + } + } + async getObjectMDStats(bucketName, bucketInfo, isTransient, log, callback) { let cursor; try { @@ -100,6 +183,9 @@ class S3UtilsMongoClient extends MongoClientInterface { account: {}, // account level metrics }; let stalledCount = 0; + let bucketKey; + let inflightsPreScan = 0; + let accountBucket; const cmpDate = new Date(); cmpDate.setHours(cmpDate.getHours() - 1); @@ -111,6 +197,14 @@ class S3UtilsMongoClient extends MongoClientInterface { return callback(errors.InternalError); } + const bucketEntry = usersBucketCreationDatesMap[`${bucketInfo.getOwner()}${constants.splitter}${bucketName}`]; + if (bucketEntry) { + bucketKey = `bucket_${bucketName}_${new Date(usersBucketCreationDatesMap[bucketEntry]).getTime()}`; + if (bucketKey) { + inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); + } + } + let startCursorDate = new Date(); let processed = 0; await cursor.forEach( @@ -234,6 +328,8 @@ class S3UtilsMongoClient extends MongoClientInterface { collRes.account[account].locations[location].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0; }); }); + // one bucket has only one account + [accountBucket] = Object.keys(collRes.account); monitoring.objectsCount.inc({ status: 'success' }); processed++; }, @@ -261,6 +357,16 @@ class S3UtilsMongoClient extends MongoClientInterface { const retResult = this._handleResults(collRes, isVer); retResult.stalled = stalledCount; + if (inflightsPreScan > 0 && retResult && retResult.dataMetrics) { + Object.keys(retResult.dataMetrics.bucket).forEach(key => { + retResult.dataMetrics.bucket[key].usedCapacity = { + ...retResult.dataMetrics.bucket[key].usedCapacity, + _inflightsPreScan: inflightsPreScan, + }; + retResult.dataMetrics.bucket[key].accountOwnerID = accountBucket; + }); + } + return callback(null, retResult); } catch (err) { log.error('An error occurred', { @@ -654,7 +760,7 @@ class S3UtilsMongoClient extends MongoClientInterface { async updateStorageConsumptionMetrics(countItems, dataMetrics, log, cb) { try { - const updatedStorageMetricsList = [ + let updatedStorageMetricsList = [ { _id: __COUNT_ITEMS, value: countItems }, // iterate every resource through dataMetrics and add to updatedStorageMetricsList ...Object.entries(dataMetrics) @@ -668,6 +774,9 @@ class S3UtilsMongoClient extends MongoClientInterface { ]; log.info('updateStorageConsumptionMetrics: updating storage metrics'); + // update the inflights + updatedStorageMetricsList = await this.updateInflightDeltas(updatedStorageMetricsList, log); + // Drop the temporary collection if it exists try { await this.getCollection(INFOSTORE_TMP).drop(); @@ -711,6 +820,24 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + async readStorageConsumptionInflights(entityName, log) { + try { + const i = this.getCollection(INFOSTORE); + const doc = await i.findOne({ _id: entityName }); + if (!doc || !doc.usedCapacity || !doc.usedCapacity._inflight) { + return 0; + } + return doc.usedCapacity._inflight; + } catch (err) { + log.error('readStorageConsumptionInflights: error reading metrics', { + error: err, + errDetails: { ...err }, + errorString: err.toString(), + }); + return 0; + } + } + /* * Overwrite the getBucketInfos method to specially handle the cases that * bucket collection exists but bucket is not in metastore collection.