From 621df0f2834f344f214d3c7f182e498fbd2e2ef1 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 6 Feb 2024 08:04:51 -0700 Subject: [PATCH 1/3] Add histogram for insertion time to completion time --- hook-worker/src/worker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 484edf7..ebb1b0f 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,6 +2,7 @@ use std::collections; use std::sync::Arc; use std::time; +use chrono::Utc; use futures::future::join_all; use hook_common::health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; @@ -234,6 +235,10 @@ async fn process_webhook_job( match send_result { Ok(_) => { + let insert_to_complete_duration = Utc::now() - webhook_job.job().created_at; + metrics::histogram!("webhook_jobs_insert_to_complete_duration_seconds", &labels) + .record((insert_to_complete_duration.num_milliseconds() as f64) / 1_000_f64); + webhook_job.complete().await.map_err(|error| { metrics::counter!("webhook_jobs_database_error", &labels).increment(1); error From eb4c3fe3712e404e930afc021a4d165900ffecd7 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 08:05:36 -0700 Subject: [PATCH 2/3] Handle feedback --- hook-worker/src/worker.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index ebb1b0f..7338275 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -6,6 +6,7 @@ use chrono::Utc; use futures::future::join_all; use hook_common::health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; +use hook_common::webhook; use hook_common::{ pgqueue::{ DatabaseError, Job, PgQueue, PgQueueJob, PgTransactionJob, RetryError, RetryInvalidError, @@ -235,15 +236,24 @@ async fn process_webhook_job( match send_result { Ok(_) => { - let insert_to_complete_duration = Utc::now() - webhook_job.job().created_at; - metrics::histogram!("webhook_jobs_insert_to_complete_duration_seconds", &labels) - .record((insert_to_complete_duration.num_milliseconds() as f64) / 1_000_f64); + let created_at = webhook_job.job().created_at; + let retries = webhook_job.job().attempt - 1; + let labels_with_retries = [ + ("queue", webhook_job.queue()), + ("retries", retries.to_string()), + ]; webhook_job.complete().await.map_err(|error| { metrics::counter!("webhook_jobs_database_error", &labels).increment(1); error })?; + let insert_to_complete_duration = Utc::now() - created_at; + metrics::histogram!( + "webhook_jobs_insert_to_complete_duration_seconds", + &labels_with_retries + ) + .record((insert_to_complete_duration.num_milliseconds() as f64) / 1_000_f64); metrics::counter!("webhook_jobs_completed", &labels).increment(1); metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) .record(elapsed); From e42bf7fff29113a051a89300b67b4670b0fbe0ee Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 08:10:57 -0700 Subject: [PATCH 3/3] Oops, spurious import --- hook-worker/src/worker.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 7338275..441e1ee 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -6,7 +6,6 @@ use chrono::Utc; use futures::future::join_all; use hook_common::health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; -use hook_common::webhook; use hook_common::{ pgqueue::{ DatabaseError, Job, PgQueue, PgQueueJob, PgTransactionJob, RetryError, RetryInvalidError,