Skip to content

Commit

Permalink
S3UTILS-148: rework logs logic and lint files
Browse files Browse the repository at this point in the history
  • Loading branch information
@williamlardier
williamlardier committed Oct 11, 2023
1 parent 3c752fd commit c130230
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 17 deletions.
5 changes: 5 additions & 0 deletions CountItems/CountManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ class CountManager {
}
const id = this.workerList.shift();
return this.workers[id].count(bucketInfo, (err, res) => {
this.log.info(`processing the bucket "${bucketInfo._name}"`, {
method: 'CountManager::_setupQueue',
bucket: bucketInfo.name,
workInQueue: this.q.length(),
});
if (err) {
return done(err);
}
Expand Down
2 changes: 1 addition & 1 deletion CountItems/CountWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class CountWorker {
}
const bucketInfo = BucketInfo.fromObj(bucketInfoObj);
const bucketName = bucketInfo.getName();
this.log.debug(`${process.pid} handling ${bucketName}`);
this.log.info(`${process.pid} handling ${bucketName}`);
return async.waterfall([
next => this.client._getIsTransient(bucketInfo, this.log, next),
(isTransient, next) => this.client.getObjectMDStats(bucketName, bucketInfo, isTransient, this.log, next),
Expand Down
5 changes: 4 additions & 1 deletion CountItems/masterProcess.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@ const CountManager = require('./CountManager');
const createMongoParams = require('../utils/createMongoParams');
const createWorkers = require('./utils/createWorkers');

const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1
? 'debug' : 'info';

const loggerConfig = {
level: 'debug',
level: logLevel,
dump: 'error',
};

Expand Down
5 changes: 4 additions & 1 deletion CountItems/workerProcess.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ const S3UtilsMongoClient = require('../utils/S3UtilsMongoClient');
const CountWorker = require('./CountWorker');
const createMongoParams = require('../utils/createMongoParams');

const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1
? 'debug' : 'info';

const loggerConfig = {
level: 'info',
level: logLevel,
dump: 'error',
};

Expand Down
49 changes: 35 additions & 14 deletions utils/S3UtilsMongoClient.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,18 @@ class S3UtilsMongoClient extends MongoClientInterface {

const usersBucketCreationDatesMap = usersBucketCreationDatesArray
.reduce((map, obj) => ({ ...map, [obj._id]: obj.value.creationDate }), {});

const startCursorDate = new Date();
let processed = 0;
await cursor.forEach(
res => {
// Periodically display information about the cursor
// if more than 30s elapsed
if (Date.now() - startCursorDate > 30000) {
log.info('Processing cursor', {
method: 'getObjectMDStats',
bucketName,
});
}
const { data, error } = this._processEntryData(
bucketName,
bucketInfo,
Expand Down Expand Up @@ -150,12 +159,14 @@ class S3UtilsMongoClient extends MongoClientInterface {
collRes.account[account].locations[location].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0;
});
});
processed++;
},
err => {
if (err) {
log.error('Error when processing mongo entries', {
method: 'getObjectMDStats',
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return callback(err);
}
Expand All @@ -178,7 +189,8 @@ class S3UtilsMongoClient extends MongoClientInterface {
} catch (err) {
log.error('An error occurred', {
method: 'getObjectMDStats',
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return callback(err);
}
Expand Down Expand Up @@ -406,8 +418,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
upsert: false,
});
if (!updateResult.ok) {
log.error(
'updateBucketCapacityInfo: failed to update bucket CapacityInfo', {
log.error('updateBucketCapacityInfo: failed to update bucket CapacityInfo', {
bucketName,
capacityInfo,
});
Expand All @@ -417,6 +428,8 @@ class S3UtilsMongoClient extends MongoClientInterface {
} catch (err) {
log.error('updateBucketCapacityInfo: error putting bucket CapacityInfo', {
error: err.message,
errDetails: { ...err },
errorString: err.toString(),
bucketName,
capacityInfo,
});
Expand Down Expand Up @@ -455,6 +468,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
...S3UtilsMongoClient.convertNumberToLong(metrics),
}))),
];
log.info('updateStorageConsumptionMetrics: updating storage metrics');

// Drop the temporary collection if it exists
try {
Expand All @@ -468,12 +482,14 @@ class S3UtilsMongoClient extends MongoClientInterface {
await tempCollection.insertMany(updatedStorageMetricsList, { ordered: false });
await async.retry(
3,
async () => tempCollection.rename(INFOSTORE, { dropTarget: true })
async () => tempCollection.rename(INFOSTORE, { dropTarget: true }),
);
return cb();
} catch (err) {
log.error('updateStorageConsumptionMetrics: error updating storage metrics', {
error: err.message,
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return cb(errors.InternalError);
}
Expand All @@ -484,15 +500,17 @@ class S3UtilsMongoClient extends MongoClientInterface {
const i = this.getCollection(INFOSTORE);
const doc = await async.retry(
3,
async () => i.findOne({ _id: entityName })
async () => i.findOne({ _id: entityName }),
);
if (!doc) {
return cb(errors.NoSuchEntity);
}
return cb(null, doc);
} catch (err) {
log.error('readStorageConsumptionMetrics: error reading metrics', {
error: err.message,
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return cb(errors.InternalError);
}
Expand All @@ -507,7 +525,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
try {
const bucketInfos = [];
const collInfos = await this.db.listCollections().toArray();
await async.eachLimit(collInfos, 10, async (value) => {
await async.eachLimit(collInfos, 10, async value => {
if (this._isSpecialCollection(value.name)) {
// skip
return;
Expand All @@ -529,7 +547,8 @@ class S3UtilsMongoClient extends MongoClientInterface {
} else {
log.error('failed to get bucket attributes', {
bucketName,
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
throw errors.InternalError;
}
Expand All @@ -542,7 +561,8 @@ class S3UtilsMongoClient extends MongoClientInterface {
} catch (err) {
log.error('could not get list of collections', {
method: '_getBucketInfos',
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return cb(err);
}
Expand Down Expand Up @@ -570,11 +590,12 @@ class S3UtilsMongoClient extends MongoClientInterface {
log.error('failed to read bucket entry from __usersbucket', {
bucketName,
ownerId,
error: err,
errDetails: { ...err },
errorString: err.toString(),
});
return cb(err);
}
}
}
}

module.exports = S3UtilsMongoClient;

0 comments on commit c130230

Please sign in to comment.