From 5918014da158abbf44540c855e35b0b5bb363fb2 Mon Sep 17 00:00:00 2001 From: Prajjwal Kumar Date: Sat, 7 Dec 2024 14:35:44 +0530 Subject: [PATCH] feat(dynamic_routing): analytics improvement using separate postgres table (#6723) Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com> --- crates/api_models/src/routing.rs | 1 - crates/common_enums/src/enums.rs | 16 ++++- .../src/dynamic_routing_stats.rs | 41 +++++++++++++ crates/diesel_models/src/enums.rs | 8 ++- crates/diesel_models/src/lib.rs | 1 + crates/diesel_models/src/query.rs | 1 + .../src/query/dynamic_routing_stats.rs | 11 ++++ crates/diesel_models/src/schema.rs | 30 ++++++++++ crates/diesel_models/src/schema_v2.rs | 30 ++++++++++ crates/router/src/core/routing/helpers.rs | 26 +++++++++ crates/router/src/db.rs | 2 + crates/router/src/db/dynamic_routing_stats.rs | 58 +++++++++++++++++++ crates/router/src/types/storage.rs | 13 +++-- .../types/storage/dynamic_routing_stats.rs | 1 + .../down.sql | 3 + .../up.sql | 26 +++++++++ 16 files changed, 256 insertions(+), 12 deletions(-) create mode 100644 crates/diesel_models/src/dynamic_routing_stats.rs create mode 100644 crates/diesel_models/src/query/dynamic_routing_stats.rs create mode 100644 crates/router/src/db/dynamic_routing_stats.rs create mode 100644 crates/router/src/types/storage/dynamic_routing_stats.rs create mode 100644 migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/down.sql create mode 100644 migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/up.sql diff --git a/crates/api_models/src/routing.rs b/crates/api_models/src/routing.rs index c4c2f072b1ea..eeb67d679ee3 100644 --- a/crates/api_models/src/routing.rs +++ b/crates/api_models/src/routing.rs @@ -740,7 +740,6 @@ pub struct ToggleDynamicRoutingPath { #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, ToSchema)] pub struct EliminationRoutingConfig { pub params: Option>, - // pub labels: Option>, pub elimination_analyser_config: Option, } diff --git a/crates/common_enums/src/enums.rs b/crates/common_enums/src/enums.rs index e94b40a4af85..7d8359800560 100644 --- a/crates/common_enums/src/enums.rs +++ b/crates/common_enums/src/enums.rs @@ -20,7 +20,9 @@ pub mod diesel_exports { DbMandateStatus as MandateStatus, DbPaymentMethodIssuerCode as PaymentMethodIssuerCode, DbPaymentType as PaymentType, DbRefundStatus as RefundStatus, DbRequestIncrementalAuthorization as RequestIncrementalAuthorization, - DbScaExemptionType as ScaExemptionType, DbWebhookDeliveryAttempt as WebhookDeliveryAttempt, + DbScaExemptionType as ScaExemptionType, + DbSuccessBasedRoutingConclusiveState as SuccessBasedRoutingConclusiveState, + DbWebhookDeliveryAttempt as WebhookDeliveryAttempt, }; } @@ -3283,10 +3285,20 @@ pub enum DeleteStatus { } #[derive( - Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize, strum::Display, Hash, + Clone, + Copy, + Debug, + Eq, + PartialEq, + serde::Deserialize, + serde::Serialize, + strum::Display, + Hash, + strum::EnumString, )] #[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] +#[router_derive::diesel_enum(storage_type = "db_enum")] pub enum SuccessBasedRoutingConclusiveState { // pc: payment connector // sc: success based routing outcome/first connector diff --git a/crates/diesel_models/src/dynamic_routing_stats.rs b/crates/diesel_models/src/dynamic_routing_stats.rs new file mode 100644 index 000000000000..168699d7f566 --- /dev/null +++ b/crates/diesel_models/src/dynamic_routing_stats.rs @@ -0,0 +1,41 @@ +use diesel::{Insertable, Queryable, Selectable}; + +use crate::schema::dynamic_routing_stats; + +#[derive(Clone, Debug, Eq, Insertable, PartialEq)] +#[diesel(table_name = dynamic_routing_stats)] +pub struct DynamicRoutingStatsNew { + pub payment_id: common_utils::id_type::PaymentId, + pub attempt_id: String, + pub merchant_id: common_utils::id_type::MerchantId, + pub profile_id: common_utils::id_type::ProfileId, + pub amount: common_utils::types::MinorUnit, + pub success_based_routing_connector: String, + pub payment_connector: String, + pub currency: Option, + pub payment_method: Option, + pub capture_method: Option, + pub authentication_type: Option, + pub payment_status: common_enums::AttemptStatus, + pub conclusive_classification: common_enums::SuccessBasedRoutingConclusiveState, + pub created_at: time::PrimitiveDateTime, +} + +#[derive(Clone, Debug, Eq, PartialEq, Queryable, Selectable, Insertable)] +#[diesel(table_name = dynamic_routing_stats, primary_key(payment_id), check_for_backend(diesel::pg::Pg))] +pub struct DynamicRoutingStats { + pub payment_id: common_utils::id_type::PaymentId, + pub attempt_id: String, + pub merchant_id: common_utils::id_type::MerchantId, + pub profile_id: common_utils::id_type::ProfileId, + pub amount: common_utils::types::MinorUnit, + pub success_based_routing_connector: String, + pub payment_connector: String, + pub currency: Option, + pub payment_method: Option, + pub capture_method: Option, + pub authentication_type: Option, + pub payment_status: common_enums::AttemptStatus, + pub conclusive_classification: common_enums::SuccessBasedRoutingConclusiveState, + pub created_at: time::PrimitiveDateTime, +} diff --git a/crates/diesel_models/src/enums.rs b/crates/diesel_models/src/enums.rs index 5de048e32ef9..19f3bf0a3823 100644 --- a/crates/diesel_models/src/enums.rs +++ b/crates/diesel_models/src/enums.rs @@ -20,9 +20,11 @@ pub mod diesel_exports { DbRefundStatus as RefundStatus, DbRefundType as RefundType, DbRequestIncrementalAuthorization as RequestIncrementalAuthorization, DbRoleScope as RoleScope, DbRoutingAlgorithmKind as RoutingAlgorithmKind, - DbScaExemptionType as ScaExemptionType, DbTotpStatus as TotpStatus, - DbTransactionType as TransactionType, DbUserRoleVersion as UserRoleVersion, - DbUserStatus as UserStatus, DbWebhookDeliveryAttempt as WebhookDeliveryAttempt, + DbScaExemptionType as ScaExemptionType, + DbSuccessBasedRoutingConclusiveState as SuccessBasedRoutingConclusiveState, + DbTotpStatus as TotpStatus, DbTransactionType as TransactionType, + DbUserRoleVersion as UserRoleVersion, DbUserStatus as UserStatus, + DbWebhookDeliveryAttempt as WebhookDeliveryAttempt, }; } pub use common_enums::*; diff --git a/crates/diesel_models/src/lib.rs b/crates/diesel_models/src/lib.rs index 7e476662ea7a..d07f84aa65e2 100644 --- a/crates/diesel_models/src/lib.rs +++ b/crates/diesel_models/src/lib.rs @@ -12,6 +12,7 @@ pub mod blocklist; pub mod blocklist_fingerprint; pub mod customers; pub mod dispute; +pub mod dynamic_routing_stats; pub mod enums; pub mod ephemeral_key; pub mod errors; diff --git a/crates/diesel_models/src/query.rs b/crates/diesel_models/src/query.rs index f966e90acba9..ab044b5c6e69 100644 --- a/crates/diesel_models/src/query.rs +++ b/crates/diesel_models/src/query.rs @@ -13,6 +13,7 @@ pub mod blocklist_fingerprint; pub mod customers; pub mod dashboard_metadata; pub mod dispute; +pub mod dynamic_routing_stats; pub mod events; pub mod file; pub mod fraud_check; diff --git a/crates/diesel_models/src/query/dynamic_routing_stats.rs b/crates/diesel_models/src/query/dynamic_routing_stats.rs new file mode 100644 index 000000000000..f6771cd103d9 --- /dev/null +++ b/crates/diesel_models/src/query/dynamic_routing_stats.rs @@ -0,0 +1,11 @@ +use super::generics; +use crate::{ + dynamic_routing_stats::{DynamicRoutingStats, DynamicRoutingStatsNew}, + PgPooledConn, StorageResult, +}; + +impl DynamicRoutingStatsNew { + pub async fn insert(self, conn: &PgPooledConn) -> StorageResult { + generics::generic_insert(conn, self).await + } +} diff --git a/crates/diesel_models/src/schema.rs b/crates/diesel_models/src/schema.rs index 936428bd46c5..c30562de9913 100644 --- a/crates/diesel_models/src/schema.rs +++ b/crates/diesel_models/src/schema.rs @@ -389,6 +389,35 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use crate::enums::diesel_exports::*; + + dynamic_routing_stats (attempt_id, merchant_id) { + #[max_length = 64] + payment_id -> Varchar, + #[max_length = 64] + attempt_id -> Varchar, + #[max_length = 64] + merchant_id -> Varchar, + #[max_length = 64] + profile_id -> Varchar, + amount -> Int8, + #[max_length = 64] + success_based_routing_connector -> Varchar, + #[max_length = 64] + payment_connector -> Varchar, + currency -> Nullable, + #[max_length = 64] + payment_method -> Nullable, + capture_method -> Nullable, + authentication_type -> Nullable, + payment_status -> AttemptStatus, + conclusive_classification -> SuccessBasedRoutingConclusiveState, + created_at -> Timestamp, + } +} + diesel::table! { use diesel::sql_types::*; use crate::enums::diesel_exports::*; @@ -1414,6 +1443,7 @@ diesel::allow_tables_to_appear_in_same_query!( customers, dashboard_metadata, dispute, + dynamic_routing_stats, events, file_metadata, fraud_check, diff --git a/crates/diesel_models/src/schema_v2.rs b/crates/diesel_models/src/schema_v2.rs index 8a1733fc986c..12bb19c0a7f1 100644 --- a/crates/diesel_models/src/schema_v2.rs +++ b/crates/diesel_models/src/schema_v2.rs @@ -401,6 +401,35 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use crate::enums::diesel_exports::*; + + dynamic_routing_stats (attempt_id, merchant_id) { + #[max_length = 64] + payment_id -> Varchar, + #[max_length = 64] + attempt_id -> Varchar, + #[max_length = 64] + merchant_id -> Varchar, + #[max_length = 64] + profile_id -> Varchar, + amount -> Int8, + #[max_length = 64] + success_based_routing_connector -> Varchar, + #[max_length = 64] + payment_connector -> Varchar, + currency -> Nullable, + #[max_length = 64] + payment_method -> Nullable, + capture_method -> Nullable, + authentication_type -> Nullable, + payment_status -> AttemptStatus, + conclusive_classification -> SuccessBasedRoutingConclusiveState, + created_at -> Timestamp, + } +} + diesel::table! { use diesel::sql_types::*; use crate::enums::diesel_exports::*; @@ -1362,6 +1391,7 @@ diesel::allow_tables_to_appear_in_same_query!( customers, dashboard_metadata, dispute, + dynamic_routing_stats, events, file_metadata, fraud_check, diff --git a/crates/router/src/core/routing/helpers.rs b/crates/router/src/core/routing/helpers.rs index 6a5e3bd0264b..112c014202e6 100644 --- a/crates/router/src/core/routing/helpers.rs +++ b/crates/router/src/core/routing/helpers.rs @@ -12,6 +12,8 @@ use api_models::routing as routing_types; use common_utils::ext_traits::ValueExt; use common_utils::{ext_traits::Encode, id_type, types::keymanager::KeyManagerState}; use diesel_models::configs; +#[cfg(all(feature = "v1", feature = "dynamic_routing"))] +use diesel_models::dynamic_routing_stats::DynamicRoutingStatsNew; #[cfg(feature = "v1")] use diesel_models::routing_algorithm; use error_stack::ResultExt; @@ -751,6 +753,23 @@ pub async fn push_metrics_with_update_window_for_success_based_routing( first_success_based_connector.to_string(), ); + let dynamic_routing_stats = DynamicRoutingStatsNew { + payment_id: payment_attempt.payment_id.to_owned(), + attempt_id: payment_attempt.attempt_id.clone(), + merchant_id: payment_attempt.merchant_id.to_owned(), + profile_id: payment_attempt.profile_id.to_owned(), + amount: payment_attempt.get_total_amount(), + success_based_routing_connector: first_success_based_connector.to_string(), + payment_connector: payment_connector.to_string(), + currency: payment_attempt.currency, + payment_method: payment_attempt.payment_method, + capture_method: payment_attempt.capture_method, + authentication_type: payment_attempt.authentication_type, + payment_status: payment_attempt.status, + conclusive_classification: outcome, + created_at: common_utils::date_time::now(), + }; + core_metrics::DYNAMIC_SUCCESS_BASED_ROUTING.add( &metrics::CONTEXT, 1, @@ -812,6 +831,13 @@ pub async fn push_metrics_with_update_window_for_success_based_routing( ); logger::debug!("successfully pushed success_based_routing metrics"); + state + .store + .insert_dynamic_routing_stat_entry(dynamic_routing_stats) + .await + .change_context(errors::ApiErrorResponse::InternalServerError) + .attach_printable("Unable to push dynamic routing stats to db")?; + client .update_success_rate( tenant_business_profile_id, diff --git a/crates/router/src/db.rs b/crates/router/src/db.rs index 1ed83be4283e..cc9a1e2f436d 100644 --- a/crates/router/src/db.rs +++ b/crates/router/src/db.rs @@ -12,6 +12,7 @@ pub mod configs; pub mod customers; pub mod dashboard_metadata; pub mod dispute; +pub mod dynamic_routing_stats; pub mod ephemeral_key; pub mod events; pub mod file; @@ -107,6 +108,7 @@ pub trait StorageInterface: + payment_method::PaymentMethodInterface + blocklist::BlocklistInterface + blocklist_fingerprint::BlocklistFingerprintInterface + + dynamic_routing_stats::DynamicRoutingStatsInterface + scheduler::SchedulerInterface + PayoutAttemptInterface + PayoutsInterface diff --git a/crates/router/src/db/dynamic_routing_stats.rs b/crates/router/src/db/dynamic_routing_stats.rs new file mode 100644 index 000000000000..d22f4fdd40b1 --- /dev/null +++ b/crates/router/src/db/dynamic_routing_stats.rs @@ -0,0 +1,58 @@ +use error_stack::report; +use router_env::{instrument, tracing}; +use storage_impl::MockDb; + +use super::Store; +use crate::{ + connection, + core::errors::{self, CustomResult}, + db::kafka_store::KafkaStore, + types::storage, +}; + +#[async_trait::async_trait] +pub trait DynamicRoutingStatsInterface { + async fn insert_dynamic_routing_stat_entry( + &self, + dynamic_routing_stat_new: storage::DynamicRoutingStatsNew, + ) -> CustomResult; +} + +#[async_trait::async_trait] +impl DynamicRoutingStatsInterface for Store { + #[instrument(skip_all)] + async fn insert_dynamic_routing_stat_entry( + &self, + dynamic_routing_stat: storage::DynamicRoutingStatsNew, + ) -> CustomResult { + let conn = connection::pg_connection_write(self).await?; + dynamic_routing_stat + .insert(&conn) + .await + .map_err(|error| report!(errors::StorageError::from(error))) + } +} + +#[async_trait::async_trait] +impl DynamicRoutingStatsInterface for MockDb { + #[instrument(skip_all)] + async fn insert_dynamic_routing_stat_entry( + &self, + _dynamic_routing_stat: storage::DynamicRoutingStatsNew, + ) -> CustomResult { + Err(errors::StorageError::MockDbError)? + } +} + +#[async_trait::async_trait] +impl DynamicRoutingStatsInterface for KafkaStore { + #[instrument(skip_all)] + async fn insert_dynamic_routing_stat_entry( + &self, + dynamic_routing_stat: storage::DynamicRoutingStatsNew, + ) -> CustomResult { + self.diesel_store + .insert_dynamic_routing_stat_entry(dynamic_routing_stat) + .await + } +} diff --git a/crates/router/src/types/storage.rs b/crates/router/src/types/storage.rs index 41ece363b4f8..24573548d797 100644 --- a/crates/router/src/types/storage.rs +++ b/crates/router/src/types/storage.rs @@ -12,6 +12,7 @@ pub mod configs; pub mod customers; pub mod dashboard_metadata; pub mod dispute; +pub mod dynamic_routing_stats; pub mod enums; pub mod ephemeral_key; pub mod events; @@ -63,12 +64,12 @@ pub use scheduler::db::process_tracker; pub use self::{ address::*, api_keys::*, authentication::*, authorization::*, blocklist::*, blocklist_fingerprint::*, blocklist_lookup::*, business_profile::*, capture::*, cards_info::*, - configs::*, customers::*, dashboard_metadata::*, dispute::*, ephemeral_key::*, events::*, - file::*, fraud_check::*, generic_link::*, gsm::*, locker_mock_up::*, mandate::*, - merchant_account::*, merchant_connector_account::*, merchant_key_store::*, payment_link::*, - payment_method::*, process_tracker::*, refund::*, reverse_lookup::*, role::*, - routing_algorithm::*, unified_translations::*, user::*, user_authentication_method::*, - user_role::*, + configs::*, customers::*, dashboard_metadata::*, dispute::*, dynamic_routing_stats::*, + ephemeral_key::*, events::*, file::*, fraud_check::*, generic_link::*, gsm::*, + locker_mock_up::*, mandate::*, merchant_account::*, merchant_connector_account::*, + merchant_key_store::*, payment_link::*, payment_method::*, process_tracker::*, refund::*, + reverse_lookup::*, role::*, routing_algorithm::*, unified_translations::*, user::*, + user_authentication_method::*, user_role::*, }; use crate::types::api::routing; diff --git a/crates/router/src/types/storage/dynamic_routing_stats.rs b/crates/router/src/types/storage/dynamic_routing_stats.rs new file mode 100644 index 000000000000..ba692a252555 --- /dev/null +++ b/crates/router/src/types/storage/dynamic_routing_stats.rs @@ -0,0 +1 @@ +pub use diesel_models::dynamic_routing_stats::{DynamicRoutingStats, DynamicRoutingStatsNew}; diff --git a/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/down.sql b/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/down.sql new file mode 100644 index 000000000000..993a082c2344 --- /dev/null +++ b/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS dynamic_routing_stats; +DROP TYPE IF EXISTS "SuccessBasedRoutingConclusiveState"; diff --git a/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/up.sql b/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/up.sql new file mode 100644 index 000000000000..6b37787d7224 --- /dev/null +++ b/migrations/2024-12-02-095127_add_new_table_dynamic_routing_stats/up.sql @@ -0,0 +1,26 @@ +--- Your SQL goes here +CREATE TYPE "SuccessBasedRoutingConclusiveState" AS ENUM( + 'true_positive', + 'false_positive', + 'true_negative', + 'false_negative' +); + +CREATE TABLE IF NOT EXISTS dynamic_routing_stats ( + payment_id VARCHAR(64) NOT NULL, + attempt_id VARCHAR(64) NOT NULL, + merchant_id VARCHAR(64) NOT NULL, + profile_id VARCHAR(64) NOT NULL, + amount BIGINT NOT NULL, + success_based_routing_connector VARCHAR(64) NOT NULL, + payment_connector VARCHAR(64) NOT NULL, + currency "Currency", + payment_method VARCHAR(64), + capture_method "CaptureMethod", + authentication_type "AuthenticationType", + payment_status "AttemptStatus" NOT NULL, + conclusive_classification "SuccessBasedRoutingConclusiveState" NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY(attempt_id, merchant_id) +); +CREATE INDEX profile_id_index ON dynamic_routing_stats (profile_id);