From 30ac3cc939a98155cdcbbd69beba8bdf92893e12 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 12 Aug 2024 13:18:55 +0200 Subject: [PATCH 1/2] Avoid duplicate countitems metrics Metrics from both workers & managers are instantiated in both kind of processes, leading to duplicates when merging (naively) the metrics. Improving merge is not trivial, for now we simply are more careful about instantiating the metrics: `bucketProcessingDuration` and `consolidationDuration` are only instantiated on master process. Since they have labels, `bucketsCount` and `objectsCount` are not affected, so we can keep them as is for now (since they are used directly from S3UtilsMongoClient, and would need a significant refactor to make it work both in prod and tests). Issue: S3UTILS-175 --- utils/monitoring.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/monitoring.js b/utils/monitoring.js index fa33098d..d982d977 100644 --- a/utils/monitoring.js +++ b/utils/monitoring.js @@ -1,22 +1,23 @@ const { errors } = require('arsenal'); const promClient = require('prom-client'); const { http } = require('httpagent'); +const cluster = require('cluster'); const aggregatorRegistry = new promClient.AggregatorRegistry(); const { collectDefaultMetrics } = promClient; // Histogram of the bucket processing duration, by the utilization service. -const bucketProcessingDuration = new promClient.Histogram({ +const bucketProcessingDuration = cluster.isMaster ? new promClient.Histogram({ name: 's3_countitems_bucket_listing_duration_seconds', help: 'Bucket processing duration', buckets: [1, 10, 60, 600, 3600, 18000, 36000], -}); +}) : null; -const consolidationDuration = new promClient.Histogram({ +const consolidationDuration = cluster.isMaster ? new promClient.Histogram({ name: 's3_countitems_bucket_merge_duration_seconds', help: 'Duration of metrics consolidation in seconds', buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10], -}); +}) : null; const bucketsCount = new promClient.Counter({ name: 's3_countitems_total_buckets_count', From c680ebfe775b4b9c6424f1b1a3a422058ef5711e Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 12 Aug 2024 14:43:49 +0200 Subject: [PATCH 2/2] Release 1.14.13 Issue: S3UTILS-175 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4c480bb0..1ca25934 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.14.12", + "version": "1.14.13", "engines": { "node": ">= 16" },