Bugfix/s3 utils 148 mongodb improvements

CountItems/CountManager.js

@@ -36,6 +36,11 @@ class CountManager {
             }
             const id = this.workerList.shift();
             return this.workers[id].count(bucketInfo, (err, res) => {
+                this.log.info('processing a bucket', {
+                    method: 'CountManager::_setupQueue',
+                    workInQueue: this.q.length(),
+                    bucketInfo,
+                });
                 if (err) {
                     return done(err);
                 }

CountItems/CountMaster.js

@@ -42,11 +42,14 @@ class CountMaster {
                     });
                     return next(err);
                 }
+                this.log.info('got buckets infos', {
+                    bucketCount: bucketList.bucketCount,
+                });
                 this.manager.addWork(bucketList);
                 return next();
             }),
             next => this.manager.start(next),
-            next => this.client.updateStorageConsumptionMetrics(this.manager.store, this.manager.dataMetrics, this._log, next),
+            next => this.client.updateStorageConsumptionMetrics(this.manager.store, this.manager.dataMetrics, this.log, next),
         ], err => {
             if (err) {
                 this.log.error('error occurred in count items job', {

CountItems/CountWorker.js

@@ -31,7 +31,7 @@ class CountWorker {
         }
         const bucketInfo = BucketInfo.fromObj(bucketInfoObj);
         const bucketName = bucketInfo.getName();
-        this.log.debug(`${process.pid} handling ${bucketName}`);
+        this.log.info(`${process.pid} handling ${bucketName}`);
         return async.waterfall([
             next => this.client._getIsTransient(bucketInfo, this.log, next),
             (isTransient, next) => this.client.getObjectMDStats(bucketName, bucketInfo, isTransient, this.log, next),

CountItems/masterProcess.js

@@ -7,8 +7,11 @@ const CountManager = require('./CountManager');
 const createMongoParams = require('../utils/createMongoParams');
 const createWorkers = require('./utils/createWorkers');
 
+const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1
+    ? 'debug' : 'info';
+
 const loggerConfig = {
-    level: 'debug',
+    level: logLevel,
     dump: 'error',
 };
 

CountItems/workerProcess.js

@@ -5,8 +5,11 @@ const S3UtilsMongoClient = require('../utils/S3UtilsMongoClient');
 const CountWorker = require('./CountWorker');
 const createMongoParams = require('../utils/createMongoParams');
 
+const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1
+    ? 'debug' : 'info';
+
 const loggerConfig = {
-    level: 'info',
+    level: logLevel,
     dump: 'error',
 };
 

StalledRetry/MongoClientInterfaceStalled.js

@@ -31,20 +31,20 @@ class MongoClientInterfaceStalled extends MongoClientInterface {
                 },
             },
             { $project: reducedFields },
-        ]);
+        ]).stream();
     }
 
-    queueStalledObjects(expiredBy, cb) {
+    async queueStalledObjects(expiredBy, cb) {
         const cmpDate = new Date();
         cmpDate.setHours(cmpDate.getHours() - expiredBy);
         const reqHandler = this.requestHandlerFactory(this.logger);
 
         let stalledCount = 0;
-        this.db.listCollections().toArray((err, collections) => {
-            if (err) {
-                return cb(err);
-            }
+        try {
+            const collections = await this.db.listCollections().toArray();
+            let i = 0;
             return async.eachSeries(collections, (value, next) => {
+                i++;
                 if (this._isSpecialCollection(value.name)) {
                     // skip
                     return next();
@@ -56,7 +56,6 @@ class MongoClientInterfaceStalled extends MongoClientInterface {
                     this._getStalledObjectsByBucket(bucketName),
                     this.logger,
                 );
-
                 return reqHandler.handleRequests(wrapper, (err, results) => {
                     if (err) {
                         this.logger.error(
@@ -76,7 +75,9 @@ class MongoClientInterfaceStalled extends MongoClientInterface {
                     return next();
                 });
             }, err => cb(err, stalledCount));
-        });
+        } catch (err) {
+            return cb(err);
+        }
     }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3utils",
-  "version": "1.14.1",
+  "version": "1.14.2",
   "engines": {
     "node": ">= 16"
   },
@@ -28,7 +28,7 @@
   "dependencies": {
     "@senx/warp10": "^1.1.2",
     "JSONStream": "^1.3.5",
-    "arsenal": "git+https://github.com/scality/arsenal#8.1.78",
+    "arsenal": "git+https://github.com/scality/arsenal#8.1.113",
     "async": "^2.6.4",
     "aws-sdk": "^2.1005.0",
     "bucketclient": "git+https://github.com/scality/bucketclient#8.1.5",

tests/conf/envSetup.js

@@ -1,2 +1,7 @@
+const { TextEncoder, TextDecoder } = require('util');
+
+global.TextEncoder = TextEncoder;
+global.TextDecoder = TextDecoder;
+
 process.env.BUCKETD_HOSTPORT = 'localhost:9000';
 process.env.SPROXYD_HOSTPORT = 'localhost';

tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js

@@ -40,7 +40,7 @@ describe('collectBucketMetricsAndUpdateBucketCapacityInfo', () => {
 
     afterAll(done => {
         async.series([
-            next => client.db.dropDatabase(next),
+            next => client.db.dropDatabase().then(() => next()).catch(() => next()),
             next => client.close(next),
         ], done);
     });

tests/functional/countItems.js

@@ -213,7 +213,7 @@ describe('CountItems', () => {
 
     afterAll(done => {
         async.series([
-            next => client.db.dropDatabase(next),
+            next => client.db.dropDatabase().then(() => next()).catch(() => next()),
             next => client.close(next),
         ], done);
     });

tests/functional/stalled.js

@@ -232,7 +232,7 @@ describe('StalledRetry', () => {
 
     afterAll(done => {
         async.series([
-            next => mgoClient.db.dropDatabase(next),
+            next => mgoClient.db.dropDatabase().then(() => next()).catch(() => next()),
             next => mgoClient.close(next),
         ], done);
     });

tests/functional/utils/S3UtilsMongoClient.js

@@ -834,19 +834,30 @@ describe('S3UtilsMongoClient::getBucketInfos', () => {
             done();
         }));
 
-        it('Should ignore buckets in collection list but not in metastore', done => {
+        it('Should ignore buckets in collection list but not in metastore', async () => {
             const metastoreCollection = client.getCollection('__metastore');
-            return metastoreCollection.deleteOne({ _id: buckets[0]._name }, err => {
-                assert.equal(err, null);
+
+            // Delete a bucket from the metastore
+            const deleteResult = await metastoreCollection.deleteOne({ _id: buckets[0]._name });
+            assert.equal(deleteResult.deletedCount, 1);
+
+            // Fetch bucket information
+            const getBucketInfosPromise = new Promise((resolve, reject) => {
                 client.getBucketInfos(logger, (err, data) => {
-                    assert.equal(err, null);
-                    assert.strictEqual(data.bucketCount, 2);
-                    assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[0]._name)), undefined);
-                    assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[1]._name)), JSON.stringify(buckets[1]));
-                    assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[2]._name)), JSON.stringify(buckets[2]));
-                    done();
+                    if (err) {
+                        return reject(err);
+                    }
+                    return resolve(data);
                 });
             });
+
+            const data = await getBucketInfosPromise;
+
+            // Perform the assertions
+            assert.strictEqual(data.bucketCount, 2);
+            assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[0]._name)), undefined);
+            assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[1]._name)), JSON.stringify(buckets[1]));
+            assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[2]._name)), JSON.stringify(buckets[2]));
         });
     });
 });