Skip to content

Commit

Permalink
Avoid duplicate countitems metrics
Browse files Browse the repository at this point in the history
Metrics from both workers & managers are instantiated in both kind of
processes, leading to duplicates when

Issue: S3UTILS-175
  • Loading branch information
francoisferrand committed Aug 12, 2024
1 parent e96a3d1 commit ab1931c
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 27 deletions.
2 changes: 1 addition & 1 deletion CountItems/CountManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const async = require('async');
const { once } = require('arsenal').jsutil;
const { validStorageMetricLevels } = require('./utils/constants');
const { consolidateDataMetrics } = require('./utils/utils');
const monitoring = require('../utils/monitoring');
const monitoring = require('../utils/monitoring').getManagerMetrics();

class CountManager {
constructor(params) {
Expand Down
2 changes: 1 addition & 1 deletion CountItems/CountWorker.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const assert = require('assert');
const async = require('async');
const { BucketInfo } = require('arsenal').models;
const monitoring = require('../utils/monitoring');
const monitoring = require('../utils/monitoring').getWorkerMetrics();

class CountWorker {
constructor(params) {
Expand Down
56 changes: 31 additions & 25 deletions utils/monitoring.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,30 +5,38 @@ const { http } = require('httpagent');
const aggregatorRegistry = new promClient.AggregatorRegistry();
const { collectDefaultMetrics } = promClient;

// Histogram of the bucket processing duration, by the utilization service.
const bucketProcessingDuration = new promClient.Histogram({
name: 's3_countitems_bucket_listing_duration_seconds',
help: 'Bucket processing duration',
buckets: [1, 10, 60, 600, 3600, 18000, 36000],
});
function getManagerMetrics() {
// Histogram of the bucket processing duration, by the utilization service.
const bucketProcessingDuration = new promClient.Histogram({
name: 's3_countitems_bucket_listing_duration_seconds',
help: 'Bucket processing duration',
buckets: [1, 10, 60, 600, 3600, 18000, 36000],
});

const consolidationDuration = new promClient.Histogram({
name: 's3_countitems_bucket_merge_duration_seconds',
help: 'Duration of metrics consolidation in seconds',
buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
});
const consolidationDuration = new promClient.Histogram({
name: 's3_countitems_bucket_merge_duration_seconds',
help: 'Duration of metrics consolidation in seconds',
buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
});

const bucketsCount = new promClient.Counter({
name: 's3_countitems_total_buckets_count',
help: 'Total number of buckets processed',
labelNames: ['status'],
});
return { bucketProcessingDuration, consolidationDuration };
}

function getWorkerMetrics() {
const bucketsCount = new promClient.Counter({
name: 's3_countitems_total_buckets_count',
help: 'Total number of buckets processed',
labelNames: ['status'],
});

const objectsCount = new promClient.Counter({
name: 's3_countitems_total_objects_count',
help: 'Total number of objects processed',
labelNames: ['status'],
});
const objectsCount = new promClient.Counter({
name: 's3_countitems_total_objects_count',
help: 'Total number of objects processed',
labelNames: ['status'],
});

return { bucketsCount, objectsCount };
}

/**
* @param {http.ServerResponse} res - http response object
Expand Down Expand Up @@ -78,8 +86,6 @@ module.exports = {
client: promClient,
collectDefaultMetrics,
metricsHandler,
bucketProcessingDuration,
consolidationDuration,
bucketsCount,
objectsCount,
getManagerMetrics,
getWorkerMetrics,
};

0 comments on commit ab1931c

Please sign in to comment.