diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index 0101f95..b70bd85 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -38,7 +38,7 @@ jobs: - name: Install latest nightly uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-11 + toolchain: 1.71.0 override: true components: rustfmt, clippy diff --git a/.gitignore b/.gitignore index 1bbc4c4..d6c7de5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ /target/ .env.local keypair.json -.vscode \ No newline at end of file +.vscode diff --git a/Cargo.lock b/Cargo.lock index b7e6b09..a6a3ea0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,6 +814,16 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1529,8 +1539,8 @@ dependencies = [ [[package]] name = "holaplex-hub-core" -version = "0.5.4" -source = "git+https://github.com/holaplex/hub-core?branch=stable#1122dc679042e397360da0f85b20babff4c7ea12" +version = "0.5.5" +source = "git+https://github.com/holaplex/hub-core#be11206ecb91d819d5ebf34c00eb8a1294f54026" dependencies = [ "anyhow", "async-trait", @@ -1545,7 +1555,11 @@ dependencies = [ "holaplex-hub-core-schemas", "hostname", "num_cpus", + "opentelemetry", + "opentelemetry-prometheus", + "opentelemetry_sdk", "pin-project-lite", + "prometheus", "prost", "prost-types", "rand 0.8.5", @@ -1584,10 +1598,29 @@ dependencies = [ "url", ] +[[package]] +name = "holaplex-hub-core-build" +version = "0.2.1" +source = "git+https://github.com/holaplex/hub-core#be11206ecb91d819d5ebf34c00eb8a1294f54026" +dependencies = [ + "anyhow", + "dotenv", + "futures-util", + "hex", + "prost-build", + "reqwest", + "serde", + "sha2 0.10.7", + "tempfile", + "tokio", + "toml 0.5.11", + "url", +] + [[package]] name = "holaplex-hub-core-macros" version = "0.1.0" -source = "git+https://github.com/holaplex/hub-core?branch=stable#1122dc679042e397360da0f85b20babff4c7ea12" +source = "git+https://github.com/holaplex/hub-core#be11206ecb91d819d5ebf34c00eb8a1294f54026" dependencies = [ "proc-macro2", "quote", @@ -1598,9 +1631,9 @@ dependencies = [ [[package]] name = "holaplex-hub-core-schemas" version = "0.3.2" -source = "git+https://github.com/holaplex/hub-core?branch=stable#1122dc679042e397360da0f85b20babff4c7ea12" +source = "git+https://github.com/holaplex/hub-core#be11206ecb91d819d5ebf34c00eb8a1294f54026" dependencies = [ - "holaplex-hub-core-build", + "holaplex-hub-core-build 0.2.1 (git+https://github.com/holaplex/hub-core)", "prost", ] @@ -1612,7 +1645,7 @@ dependencies = [ "async-graphql-poem", "async-trait", "holaplex-hub-core", - "holaplex-hub-core-build", + "holaplex-hub-core-build 0.2.1 (git+https://github.com/holaplex/hub-core?branch=stable)", "poem", "prost", "prost-types", @@ -2334,6 +2367,67 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-prometheus" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d81bc254e2d572120363a2b16cdb0d715d301b5789be0cfc26ad87e4e10e53" +dependencies = [ + "once_cell", + "opentelemetry_api", + "opentelemetry_sdk", + "prometheus", + "protobuf", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float", + "percent-encoding", + "rand 0.8.5", + "regex", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "3.9.1" @@ -2675,6 +2769,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + [[package]] name = "prost" version = "0.11.9" @@ -2729,6 +2838,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "ptr_meta" version = "0.1.4" @@ -4434,6 +4549,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" diff --git a/Cargo.toml b/Cargo.toml index 6a72d43..3b2f56a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] members = ["api", "migration"] -resolver = "2" \ No newline at end of file +resolver = "2" diff --git a/api/Cargo.toml b/api/Cargo.toml index c75d0ec..1d22fc5 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -42,10 +42,9 @@ strum = { version = "0.24.1", features = ["derive"] } [dependencies.hub-core] package = "holaplex-hub-core" -version = "0.5.4" +version = "0.5.5" git = "https://github.com/holaplex/hub-core" -branch = "stable" -features = ["kafka", "credits", "asset_proxy", "sea-orm"] +features = ["kafka", "credits", "asset_proxy", "sea-orm", "metrics"] [build-dependencies.hub-core-build] package = "holaplex-hub-core-build" diff --git a/api/src/events.rs b/api/src/events.rs index 7da54e5..bd73902 100644 --- a/api/src/events.rs +++ b/api/src/events.rs @@ -1,6 +1,7 @@ use hub_core::{ chrono::{DateTime, NaiveDateTime, Offset, Utc}, credits::{CreditsClient, TransactionId}, + metrics::KeyValue, prelude::*, producer::Producer, thiserror, @@ -26,6 +27,7 @@ use crate::{ sea_orm_active_enums::{Blockchain, CreationStatus}, switch_collection_histories, transfer_charges, update_histories, }, + metrics::Metrics, proto::{ nft_events::Event as NftEvent, polygon_nft_events::Event as PolygonNftEvents, @@ -113,6 +115,7 @@ pub struct Processor { pub db: Connection, pub credits: CreditsClient, pub producer: Producer, + pub metrics: Metrics, } #[derive(Clone)] @@ -151,16 +154,17 @@ impl Processor { db: Connection, credits: CreditsClient, producer: Producer, + metrics: Metrics, ) -> Self { Self { db, credits, producer, + metrics, } } #[allow(clippy::too_many_lines)] - /// Processes incoming messages related to different services like Treasury and Solana. /// Routes each message to the corresponding handler based on the type of service and the specific event. @@ -817,6 +821,17 @@ impl Processor { creation_status = NftCreationStatus::Failed; } + let now = Utc::now(); + let elapsed = now + .signed_duration_since(collection_mint.created_at) + .num_milliseconds(); + self.metrics + .mint_duration_ms_bucket + .record(elapsed, &[KeyValue::new( + "status", + creation_status.as_str_name(), + )]); + self.producer .send( Some(&NftEvents { @@ -863,7 +878,8 @@ impl Processor { } async fn mint_updated(&self, id: String, payload: UpdateResult) -> ProcessResult<()> { - let update_history = UpdateHistories::find_by_id(Uuid::from_str(&id)?) + let id: Uuid = id.parse()?; + let update_history = UpdateHistories::find_by_id(id) .one(self.db.get()) .await? .ok_or(ProcessorErrorKind::DbMissingUpdateHistory)?; diff --git a/api/src/handlers.rs b/api/src/handlers.rs index d1a32bb..5c3aa37 100644 --- a/api/src/handlers.rs +++ b/api/src/handlers.rs @@ -1,16 +1,22 @@ use async_graphql::http::{playground_source, GraphQLPlaygroundConfig}; use async_graphql_poem::{GraphQLRequest, GraphQLResponse}; -use hub_core::anyhow::Result; +use hub_core::{ + anyhow::Result, + metrics::{Encoder, TextEncoder}, +}; use poem::{ handler, + http::StatusCode, web::{Data, Html}, IntoResponse, }; -use crate::{AppContext, AppState, Balance, OrganizationId, UserID}; +use crate::{AppContext, AppState, Balance, Metrics, OrganizationId, UserID}; #[handler] -pub fn health() {} +pub fn health() -> StatusCode { + StatusCode::OK +} #[handler] pub fn playground() -> impl IntoResponse { @@ -42,3 +48,11 @@ pub async fn graphql_handler( .await .into()) } + +#[handler] +pub fn metrics_handler(Data(metrics): Data<&Metrics>) -> Result { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + encoder.encode(&metrics.registry.gather(), &mut buffer)?; + Ok(String::from_utf8_lossy(&buffer).into_owned()) +} diff --git a/api/src/lib.rs b/api/src/lib.rs index 7254b3c..6ad411d 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -10,6 +10,7 @@ pub mod entities; pub mod events; pub mod handlers; pub mod metadata_json; +pub mod metrics; pub mod mutations; pub mod nft_storage; pub mod objects; @@ -41,6 +42,7 @@ use hub_core::{ tokio, uuid::Uuid, }; +use metrics::Metrics; use mutations::Mutation; use nft_storage::NftStorageClient; use poem::{async_trait, FromRequest, Request, RequestBody}; @@ -239,6 +241,7 @@ pub struct AppState { impl AppState { #[must_use] + #[allow(clippy::too_many_arguments)] pub fn new( schema: AppSchema, connection: Connection, @@ -292,6 +295,7 @@ pub struct AppContext { impl AppContext { #[must_use] + #[allow(clippy::similar_names)] pub fn new( db: Connection, user_id: UserID, @@ -338,6 +342,7 @@ impl AppContext { DataLoader::new(CollectionMintTransfersLoader::new(db.clone()), tokio::spawn); let switch_collection_history_loader = DataLoader::new(SwitchCollectionHistoryLoader::new(db.clone()), tokio::spawn); + Self { db, user_id, diff --git a/api/src/main.rs b/api/src/main.rs index 15dcc7c..b7c3369 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -5,7 +5,8 @@ use holaplex_hub_nfts::{ build_schema, db::Connection, events, - handlers::{graphql_handler, health, playground}, + handlers::{graphql_handler, health, metrics_handler, playground}, + metrics::Metrics, nft_storage::NftStorageClient, proto, Actions, AppState, Args, Services, }; @@ -38,8 +39,15 @@ pub fn main() { .await?; let credits = common.credits_cfg.build::().await?; let nft_storage = NftStorageClient::new(nft_storage)?; - let event_processor = - events::Processor::new(connection.clone(), credits.clone(), producer.clone()); + + let metrics = Metrics::new()?; + + let event_processor = events::Processor::new( + connection.clone(), + credits.clone(), + producer.clone(), + metrics.clone(), + ); let solana = Solana::new(producer.clone()); let polygon = Polygon::new(producer.clone()); @@ -72,9 +80,13 @@ pub fn main() { Server::new(TcpListener::bind(format!("0.0.0.0:{port}"))) .run( Route::new() - .at("/graphql", post(graphql_handler).with(AddData::new(state))) + .at( + "/graphql", + post(graphql_handler).with(AddData::new(state.clone())), + ) .at("/playground", get(playground)) - .at("/health", get(health)), + .at("/health", get(health)) + .at("/metrics", get(metrics_handler).with(AddData::new(metrics))), ) .await .context("failed to build graphql server") diff --git a/api/src/metrics.rs b/api/src/metrics.rs new file mode 100644 index 0000000..9533325 --- /dev/null +++ b/api/src/metrics.rs @@ -0,0 +1,47 @@ +#[allow(clippy::wildcard_imports)] +use hub_core::{ + anyhow::{anyhow, Result}, + metrics::*, +}; + +#[derive(Clone)] +pub struct Metrics { + pub registry: Registry, + pub provider: MeterProvider, + pub mint_duration_ms_bucket: Histogram, +} + +impl Metrics { + /// Res + /// # Errors + pub fn new() -> Result { + let registry = Registry::new(); + let exporter = hub_core::metrics::exporter() + .with_registry(registry.clone()) + .with_namespace("hub_nfts") + .build() + .map_err(|e| anyhow!("Failed to build exporter: {}", e))?; + + let provider = MeterProvider::builder() + .with_reader(exporter) + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "hub-nfts", + )])) + .build(); + + let meter = provider.meter("hub-nfts"); + + let mint_duration_ms_bucket = meter + .i64_histogram("mint.time") + .with_unit(Unit::new("ms")) + .with_description("Mint duration time in milliseconds.") + .init(); + + Ok(Self { + registry, + provider, + mint_duration_ms_bucket, + }) + } +} diff --git a/api/src/mutations/mint.rs b/api/src/mutations/mint.rs index 71b6f28..76085c5 100644 --- a/api/src/mutations/mint.rs +++ b/api/src/mutations/mint.rs @@ -744,7 +744,8 @@ impl Mutation { }) } - // Retries a mint which failed by passing its ID. + /// Retries a mint which failed by passing its ID. + /// # Errors pub async fn retry_mint_to_collection( &self, ctx: &Context<'_>, diff --git a/migration/Cargo.toml b/migration/Cargo.toml index c3ae2b0..6ab8d30 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -22,11 +22,11 @@ name = "migration" [dependencies] -tokio = { version = "1.22.0", features = ["macros"] } +tokio = { version = "1.32.0", features = ["macros"] } [dependencies.sea-orm-migration] version = "0.12.2" features = [ "runtime-tokio-rustls", - "sqlx-postgres", -] + "sqlx-postgres", +] \ No newline at end of file