Add metrics for lifecycle lateness

Issue: BB-447
scality · Nov 22, 2023 · c3663a0 · c3663a0
1 parent 94e19de
commit c3663a0
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 4 deletions.
diff --git a/extensions/lifecycle/LifecycleMetrics.js b/extensions/lifecycle/LifecycleMetrics.js
@@ -49,6 +49,13 @@ const lifecycleTriggerLatency = ZenkoMetrics.createHistogram({
     buckets: [60, 600, 3600, 2 * 3600, 4 * 3600, 8 * 3600, 16 * 3600, 24 * 3600, 48 * 3600],
 });
 
+const lifecycleLateness = ZenkoMetrics.createHistogram({
+    name: 's3_lifecycle_lateness_seconds',
+    help: 'Delay between the theoretical date and start of the lifecycle operation processing',
+    labelNames: [LIFECYCLE_LABEL_TYPE, LIFECYCLE_LABEL_LOCATION],
+    buckets: [60, 600, 3600, 2 * 3600, 4 * 3600, 8 * 3600, 16 * 3600, 24 * 3600, 48 * 3600],
+});
+
 const lifecycleDuration = ZenkoMetrics.createHistogram({
     name: 's3_lifecycle_duration_seconds',
     help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' +
@@ -118,6 +125,17 @@ class LifecycleMetrics {
         }
     }
 
+    static onLifecycleStarted(log, type, location, durationMs) {
+        try {
+            lifecycleLateness.observe({
+                [LIFECYCLE_LABEL_TYPE]: type,
+                [LIFECYCLE_LABEL_LOCATION]: location,
+            }, durationMs / 1000);
+        } catch (err) {
+            LifecycleMetrics.handleError(log, err, 'LifecycleMetrics.onLifecycleStarted');
+        }
+    }
+
     static onLifecycleCompleted(log, type, location, durationMs) {
         try {
             lifecycleDuration.observe({

diff --git a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js
@@ -98,7 +98,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
         return done();
     }
 
-    _executeDelete(entry, log, done) {
+    _executeDelete(entry, startTime, log, done) {
         const { accountId } = entry.getAttribute('target');
 
         const reqParams = {
@@ -112,6 +112,8 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
         let reqMethod;
 
         const actionType = entry.getActionType();
+        const transitionTime = entry.getAttribute('transitionTime');
+        const location = entry.getAttribute('details.dataStoreName');
         let req = null;
         if (actionType === 'deleteObject') {
             reqMethod = 'deleteObject';
@@ -121,6 +123,10 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
                 return done(errors.InternalError
                     .customizeDescription('Unable to obtain client'));
             }
+
+            LifecycleMetrics.onLifecycleStarted(log, 'expiration',
+                location, startTime - transitionTime);
+
             req = backbeatClient.deleteObjectFromExpiration(reqParams);
         } else if (actionType === 'deleteMPU') {
             reqParams.UploadId = entry.getAttribute('details.UploadId');
@@ -131,15 +137,18 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
                 return done(errors.InternalError
                     .customizeDescription('Unable to obtain client'));
             }
+
+            LifecycleMetrics.onLifecycleStarted(log, 'expiration:mpu',
+                location, startTime - transitionTime);
+
             req = s3Client[reqMethod](reqParams);
         }
         attachReqUids(req, log);
         return req.send(err => {
             LifecycleMetrics.onS3Request(log, reqMethod, 'expiration', err);
             LifecycleMetrics.onLifecycleCompleted(log,
                 actionType === 'deleteMPU' ? 'expiration:mpu' : 'expiration',
-                entry.getAttribute('details.dataStoreName'),
-                Date.now() - entry.getAttribute('transitionTime'));
+                location, Date.now() - entry.getAttribute('transitionTime'));
 
             if (err) {
                 log.error(
@@ -226,6 +235,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
      */
 
     processActionEntry(entry, done) {
+        const startTime = Date.now();
         const log = this.logger.newRequestLogger();
         entry.addLoggedAttributes({
             bucketName: 'target.bucket',
@@ -237,7 +247,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
             next => this._checkDate(entry, log, next),
             next => this._checkObjectLockState(entry, log, next),
             next => this._checkReplicationStatus(entry, log, next),
-            next => this._executeDelete(entry, log, next),
+            next => this._executeDelete(entry, startTime, log, next),
         ], err => {
             if (err && err instanceof ObjectLockedError) {
                 log.debug('Object is locked, skipping',

diff --git a/extensions/lifecycle/tasks/LifecycleUpdateExpirationTask.js b/extensions/lifecycle/tasks/LifecycleUpdateExpirationTask.js
@@ -139,6 +139,7 @@ class LifecycleUpdateExpirationTask extends BackbeatTask {
      * @return {undefined}
      */
     processActionEntry(entry, done) {
+        const startTime = Date.now();
         const log = this.logger.newRequestLogger();
         entry.addLoggedAttributes({
             bucketName: 'target.bucket',
@@ -164,6 +165,9 @@ class LifecycleUpdateExpirationTask extends BackbeatTask {
                     return process.nextTick(done);
                 }
 
+                LifecycleMetrics.onLifecycleStarted(log, 'restore:delete',
+                    coldLocation, startTime - restoreExpirationDate);
+
                 // Reset archive flags to no longer show it as restored
                 objMD.setArchive({
                     archiveInfo: archive.archiveInfo,

diff --git a/extensions/replication/tasks/CopyLocationTask.js b/extensions/replication/tasks/CopyLocationTask.js
@@ -7,6 +7,7 @@ const { ObjectMD } = models;
 const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy');
 const BackbeatClient = require('../../../lib/clients/BackbeatClient');
 const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
+const LifecycleMetrics = require('../../lifecycle/LifecycleMetrics');
 const ReplicationMetric = require('../ReplicationMetric');
 const ReplicationMetrics = require('../ReplicationMetrics');
 const { attachReqUids } = require('../../../lib/clients/utils');
@@ -87,6 +88,7 @@ class CopyLocationTask extends BackbeatTask {
     }
 
     processQueueEntry(actionEntry, kafkaEntry, done) {
+        const startTime = Date.now();
         const log = this.logger.newRequestLogger();
         actionEntry.addLoggedAttributes({
             bucketName: 'target.bucket',
@@ -119,6 +121,11 @@ class CopyLocationTask extends BackbeatTask {
                 if (err) {
                     return next(err);
                 }
+
+                LifecycleMetrics.onLifecycleStarted(log, 'transition',
+                    actionEntry.getAttribute('toLocation'),
+                    startTime - Date.parse(objMD.getTransitionTime));
+
                 // Do a multipart upload when either the size is above
                 // a threshold or the source object is itself a MPU.
                 //