From 85a077e80daad53b0b5475d626fc91f46f950216 Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Sun, 26 May 2024 21:36:03 +0100 Subject: [PATCH 1/2] refactor: migrate to tracing for logging --- .gitignore | 3 + .rustfmt.toml | 3 + Cargo.lock | 242 ++++++++++++++++++-------------- Cargo.toml | 8 +- build.rs | 4 +- libs/badge/badge.rs | 4 +- src/engine/fut/crawl.rs | 11 +- src/engine/fut/mod.rs | 3 +- src/engine/machines/analyzer.rs | 3 +- src/engine/machines/crawler.rs | 9 +- src/engine/mod.rs | 36 +++-- src/interactors/crates.rs | 6 +- src/interactors/github.rs | 1 - src/interactors/rustsec.rs | 6 +- src/main.rs | 57 +++++--- src/parsers/manifest.rs | 3 +- src/server/mod.rs | 61 ++++---- src/server/views/badge.rs | 6 +- src/server/views/html/index.rs | 8 +- src/server/views/html/mod.rs | 6 +- src/server/views/html/status.rs | 15 +- src/utils/cache.rs | 21 ++- src/utils/index.rs | 46 +++--- 23 files changed, 300 insertions(+), 262 deletions(-) create mode 100644 .rustfmt.toml diff --git a/.gitignore b/.gitignore index f2e972d..a31a4f3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ # These are backup files generated by rustfmt **/*.rs.bk + +# local environment variables +.env diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..efd3c5b --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,3 @@ +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +use_field_init_shorthand = true diff --git a/Cargo.lock b/Cargo.lock index 9721623..66a9d16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -144,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata", + "regex-automata 0.4.6", "serde", ] @@ -380,25 +389,10 @@ dependencies = [ ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "dotenvy" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dunce" @@ -437,6 +431,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error_reporter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8" + [[package]] name = "faster-hex" version = "0.9.0" @@ -1673,17 +1673,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "is-terminal" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "itoa" version = "1.0.11" @@ -1739,16 +1728,6 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.5.0", - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1777,6 +1756,15 @@ version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "maud" version = "0.26.0" @@ -1869,6 +1857,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -1953,6 +1951,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "owned_ttf_parser" version = "0.15.2" @@ -2206,14 +2210,24 @@ dependencies = [ ] [[package]] -name = "redox_users" -version = "0.4.5" +name = "regex" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ - "getrandom", - "libredox", - "thiserror", + "aho-corasick", + "memchr", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2221,6 +2235,23 @@ name = "regex-automata" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "relative-path" @@ -2422,12 +2453,6 @@ dependencies = [ "owned_ttf_parser", ] -[[package]] -name = "rustversion" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" - [[package]] name = "ryu" version = "1.0.18" @@ -2559,6 +2584,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -2574,6 +2608,8 @@ dependencies = [ "cadence", "crates-index", "derive_more", + "dotenvy", + "error_reporter", "font-awesome-as-a-crate", "futures-util", "gix", @@ -2583,6 +2619,7 @@ dependencies = [ "lru_time_cache", "maud", "once_cell", + "parking_lot", "pulldown-cmark", "relative-path", "reqwest", @@ -2592,11 +2629,10 @@ dependencies = [ "serde", "serde_urlencoded", "sha-1", - "slog", - "slog-async", - "slog-term", "tokio", "toml 0.8.13", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2614,37 +2650,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-term" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" -dependencies = [ - "is-terminal", - "slog", - "term", - "thread_local", - "time", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -2737,12 +2742,6 @@ dependencies = [ "libc", ] -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - [[package]] name = "tame-index" version = "0.12.0" @@ -2780,17 +2779,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "thiserror" version = "1.0.61" @@ -3033,9 +3021,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -3043,6 +3043,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3142,6 +3172,12 @@ dependencies = [ "serde", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 746daaf..607ec65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,16 @@ anyhow = "1" cadence = "1" crates-index = { version = "2", default-features = false, features = ["git"] } derive_more = "0.99" +dotenvy = "0.15" font-awesome-as-a-crate = "0.3" futures-util = { version = "0.3", default-features = false, features = ["std"] } hyper = { version = "0.14.10", features = ["full"] } +error_reporter = "1" indexmap = { version = "2", features = ["serde"] } lru_time_cache = "0.11" maud = "0.26" once_cell = "1" +parking_lot = "0.12" pulldown-cmark = "0.11" relative-path = { version = "1", features = ["serde"] } reqwest = { version = "0.12", features = ["json"] } @@ -33,11 +36,10 @@ rustsec = "0.29" semver = { version = "1.0", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_urlencoded = "0.7" -slog = "2" -slog-async = "2" -slog-term = "2" tokio = { version = "1.24.2", features = ["rt-multi-thread", "macros", "sync", "time"] } toml = "0.8" +tracing = "0.1.30" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } [target.'cfg(any())'.dependencies] gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] } diff --git a/build.rs b/build.rs index 6ac4147..f8e3f0d 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,4 @@ -use std::env; -use std::fs; -use std::path::Path; +use std::{env, fs, path::Path}; use sha1::{Digest, Sha1}; diff --git a/libs/badge/badge.rs b/libs/badge/badge.rs index 146eb69..f328738 100644 --- a/libs/badge/badge.rs +++ b/libs/badge/badge.rs @@ -235,8 +235,8 @@ mod tests { #[test] #[ignore] fn test_to_svg() { - use std::fs::File; - use std::io::Write; + use std::{fs::File, io::Write as _}; + let mut file = File::create("test.svg").unwrap(); let options = BadgeOptions { subject: "latest".to_owned(), diff --git a/src/engine/fut/crawl.rs b/src/engine/fut/crawl.rs index 08165dc..c30d88a 100644 --- a/src/engine/fut/crawl.rs +++ b/src/engine/fut/crawl.rs @@ -2,11 +2,12 @@ use anyhow::Error; use futures_util::{future::BoxFuture, stream::FuturesOrdered, FutureExt as _, StreamExt as _}; use relative_path::RelativePathBuf; -use crate::models::repo::RepoPath; - -use crate::engine::{ - machines::crawler::{ManifestCrawler, ManifestCrawlerOutput}, - Engine, +use crate::{ + engine::{ + machines::crawler::{ManifestCrawler, ManifestCrawlerOutput}, + Engine, + }, + models::repo::RepoPath, }; pub async fn crawl_manifest( diff --git a/src/engine/fut/mod.rs b/src/engine/fut/mod.rs index 9112e18..800118d 100644 --- a/src/engine/fut/mod.rs +++ b/src/engine/fut/mod.rs @@ -1,5 +1,4 @@ mod analyze; mod crawl; -pub use self::analyze::analyze_dependencies; -pub use self::crawl::crawl_manifest; +pub use self::{analyze::analyze_dependencies, crawl::crawl_manifest}; diff --git a/src/engine/machines/analyzer.rs b/src/engine/machines/analyzer.rs index 8ebfb01..4e607c9 100644 --- a/src/engine/machines/analyzer.rs +++ b/src/engine/machines/analyzer.rs @@ -101,9 +101,8 @@ impl DependencyAnalyzer { #[cfg(test)] mod tests { - use crate::models::crates::{CrateDep, CrateDeps, CrateRelease}; - use super::*; + use crate::models::crates::{CrateDep, CrateDeps, CrateRelease}; #[test] fn tracks_latest_without_matching() { diff --git a/src/engine/machines/crawler.rs b/src/engine/machines/crawler.rs index 8eb7ba4..267db8d 100644 --- a/src/engine/machines/crawler.rs +++ b/src/engine/machines/crawler.rs @@ -4,8 +4,10 @@ use anyhow::Error; use indexmap::IndexMap; use relative_path::RelativePathBuf; -use crate::models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName}; -use crate::parsers::manifest::parse_manifest_toml; +use crate::{ + models::crates::{CrateDep, CrateDeps, CrateManifest, CrateName}, + parsers::manifest::parse_manifest_toml, +}; pub struct ManifestCrawlerOutput { pub crates: IndexMap, @@ -118,9 +120,8 @@ mod tests { use relative_path::RelativePath; use semver::VersionReq; - use crate::models::crates::CrateDep; - use super::*; + use crate::models::crates::CrateDep; #[test] fn simple_package_manifest() { diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 2603ab3..34e89ea 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -7,7 +7,6 @@ use std::{ use anyhow::{anyhow, Error}; use cadence::{MetricSink, NopMetricSink, StatsdClient}; - use futures_util::{ future::try_join_all, stream::{self, BoxStream}, @@ -18,16 +17,21 @@ use once_cell::sync::Lazy; use relative_path::{RelativePath, RelativePathBuf}; use rustsec::database::Database; use semver::VersionReq; -use slog::Logger; -use crate::interactors::crates::{GetPopularCrates, QueryCrate}; -use crate::interactors::github::GetPopularRepos; -use crate::interactors::rustsec::FetchAdvisoryDatabase; -use crate::interactors::RetrieveFileAtPath; -use crate::models::crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease}; -use crate::models::repo::{RepoPath, Repository}; -use crate::utils::cache::Cache; -use crate::ManagedIndex; +use crate::{ + interactors::{ + crates::{GetPopularCrates, QueryCrate}, + github::GetPopularRepos, + rustsec::FetchAdvisoryDatabase, + RetrieveFileAtPath, + }, + models::{ + crates::{AnalyzedDependencies, CrateName, CratePath, CrateRelease}, + repo::{RepoPath, Repository}, + }, + utils::cache::Cache, + ManagedIndex, +}; mod fut; mod machines; @@ -45,33 +49,25 @@ pub struct Engine { } impl Engine { - pub fn new(client: reqwest::Client, index: ManagedIndex, logger: Logger) -> Engine { + pub fn new(client: reqwest::Client, index: ManagedIndex) -> Engine { let metrics = Arc::new(StatsdClient::from_sink("engine", NopMetricSink)); - let query_crate = Cache::new( - QueryCrate::new(index), - Duration::from_secs(10), - 500, - logger.clone(), - ); + let query_crate = Cache::new(QueryCrate::new(index), Duration::from_secs(10), 500); let get_popular_crates = Cache::new( GetPopularCrates::new(client.clone()), Duration::from_secs(15 * 60), 1, - logger.clone(), ); let get_popular_repos = Cache::new( GetPopularRepos::new(client.clone()), Duration::from_secs(5 * 60), 1, - logger.clone(), ); let retrieve_file_at_path = RetrieveFileAtPath::new(client.clone()); let fetch_advisory_db = Cache::new( FetchAdvisoryDatabase::new(client), Duration::from_secs(30 * 60), 1, - logger, ); Engine { diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index e490a64..1a07f5c 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -1,4 +1,7 @@ -use std::{fmt, str, task::Context, task::Poll}; +use std::{ + fmt, str, + task::{Context, Poll}, +}; use anyhow::{anyhow, Error}; use crates_index::{Crate, DependencyKind}; @@ -6,7 +9,6 @@ use futures_util::FutureExt as _; use hyper::service::Service; use semver::{Version, VersionReq}; use serde::Deserialize; - use tokio::task::spawn_blocking; use crate::{ diff --git a/src/interactors/github.rs b/src/interactors/github.rs index 32d5c86..72bb684 100644 --- a/src/interactors/github.rs +++ b/src/interactors/github.rs @@ -4,7 +4,6 @@ use std::{ }; use anyhow::Error; - use futures_util::FutureExt as _; use hyper::service::Service; use serde::Deserialize; diff --git a/src/interactors/rustsec.rs b/src/interactors/rustsec.rs index e28488e..46ddd55 100644 --- a/src/interactors/rustsec.rs +++ b/src/interactors/rustsec.rs @@ -1,4 +1,8 @@ -use std::{fmt, sync::Arc, task::Context, task::Poll}; +use std::{ + fmt, + sync::Arc, + task::{Context, Poll}, +}; use anyhow::Error; use futures_util::FutureExt as _; diff --git a/src/main.rs b/src/main.rs index c568f0b..c5414ac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,9 +15,8 @@ use hyper::{ service::{make_service_fn, service_fn}, Server, }; - use reqwest::redirect::Policy as RedirectPolicy; -use slog::{error, info, o, Drain, Logger}; +use tracing::Instrument as _; mod engine; mod interactors; @@ -26,9 +25,7 @@ mod parsers; mod server; mod utils; -use self::engine::Engine; -use self::server::App; -use self::utils::index::ManagedIndex; +use self::{engine::Engine, server::App, utils::index::ManagedIndex}; /// Future crate's BoxFuture without the explicit lifetime parameter. pub type BoxFuture = Pin + Send>>; @@ -43,18 +40,29 @@ fn init_metrics() -> QueuingMetricSink { QueuingMetricSink::from(sink) } -fn init_root_logger() -> Logger { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - - Logger::root(drain, o!()) +fn init_tracing_subscriber() { + use tracing::level_filters::LevelFilter; + use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + + let stdout_logger = match env::var("RUST_LOG_TIME").as_deref() { + Ok("false") => fmt::layer().without_time().boxed(), + _ => fmt::layer().boxed(), + }; + + tracing_subscriber::registry() + .with( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .with(stdout_logger) + .init(); } #[tokio::main] async fn main() { - let logger = init_root_logger(); - + dotenvy::dotenv().ok(); + init_tracing_subscriber(); let metrics = init_metrics(); let client = reqwest::Client::builder() @@ -71,7 +79,7 @@ async fn main() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); - let index = ManagedIndex::new(logger.clone()); + let index = ManagedIndex::new(); { let index = index.clone(); @@ -81,27 +89,32 @@ async fn main() { }); } - let mut engine = Engine::new(client.clone(), index, logger.new(o!())); + let mut engine = Engine::new(client.clone(), index); engine.set_metrics(metrics); - let svc_logger = logger.new(o!()); let make_svc = make_service_fn(move |_socket: &AddrStream| { let engine = engine.clone(); - let logger = svc_logger.clone(); async move { - let server = App::new(logger.clone(), engine.clone()); + let server = App::new(engine.clone()); Ok::<_, hyper::Error>(service_fn(move |req| { let server = server.clone(); - async move { server.handle(req).await } + async move { + let path = req.uri().path().to_owned(); + + server + .handle(req) + .instrument(tracing::info_span!("@", %path)) + .await + } })) } }); let server = Server::bind(&addr).serve(make_svc); - info!(logger, "Server running on port {}", port); + tracing::info!("Server running on port {port}"); - if let Err(e) = server.await { - error!(logger, "server error: {}", e); + if let Err(err) = server.await { + tracing::error!("server error: {err}"); } } diff --git a/src/parsers/manifest.rs b/src/parsers/manifest.rs index 64535fd..c461050 100644 --- a/src/parsers/manifest.rs +++ b/src/parsers/manifest.rs @@ -135,9 +135,8 @@ pub fn parse_manifest_toml(input: &str) -> Result { #[cfg(test)] mod tests { - use crate::models::crates::CrateManifest; - use super::*; + use crate::models::crates::CrateManifest; #[test] fn parse_workspace_without_members_declaration() { diff --git a/src/server/mod.rs b/src/server/mod.rs index ea54c19..3355066 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -10,7 +10,6 @@ use once_cell::sync::Lazy; use route_recognizer::{Params, Router}; use semver::VersionReq; use serde::Deserialize; -use slog::{error, info, o, Logger}; mod assets; mod views; @@ -18,10 +17,14 @@ mod views; use self::assets::{ STATIC_LINKS_JS_ETAG, STATIC_LINKS_JS_PATH, STATIC_STYLE_CSS_ETAG, STATIC_STYLE_CSS_PATH, }; -use crate::engine::{AnalyzeDependenciesOutcome, Engine}; -use crate::models::crates::{CrateName, CratePath}; -use crate::models::repo::RepoPath; -use crate::models::SubjectPath; +use crate::{ + engine::{AnalyzeDependenciesOutcome, Engine}, + models::{ + crates::{CrateName, CratePath}, + repo::RepoPath, + SubjectPath, + }, +}; #[derive(Debug, Clone, Copy, PartialEq)] enum StatusFormat { @@ -47,13 +50,12 @@ enum Route { #[derive(Clone)] pub struct App { - logger: Logger, engine: Engine, router: Arc>, } impl App { - pub fn new(logger: Logger, engine: Engine) -> App { + pub fn new(engine: Engine) -> App { let mut router = Router::new(); router.add("/", Route::Index); @@ -83,15 +85,12 @@ impl App { ); App { - logger, engine, router: Arc::new(router), } } pub async fn handle(&self, req: Request) -> Result, HyperError> { - let logger = self.logger.new(o!("path" => req.uri().path().to_owned())); - let logger2 = logger.clone(); let start = Instant::now(); // allows `/path/` to also match `/path` @@ -99,28 +98,25 @@ impl App { let res = if let Ok(route_match) = self.router.recognize(normalized_path) { match (req.method(), route_match.handler()) { - (&Method::GET, Route::Index) => { - self.index(req, route_match.params().clone(), logger).await - } + (&Method::GET, Route::Index) => self.index(req, route_match.params().clone()).await, (&Method::GET, Route::RepoStatus(format)) => { - self.repo_status(req, route_match.params().clone(), logger, *format) + self.repo_status(req, route_match.params().clone(), *format) .await } (&Method::GET, Route::CrateStatus(format)) => { - self.crate_status(req, route_match.params().clone(), logger, *format) + self.crate_status(req, route_match.params().clone(), *format) .await } (&Method::GET, Route::LatestCrateBadge) => { - self.crate_status(req, route_match.params().clone(), logger, StatusFormat::Svg) + self.crate_status(req, route_match.params().clone(), StatusFormat::Svg) .await } (&Method::GET, Route::CrateRedirect) => { - self.crate_redirect(req, route_match.params().clone(), logger) - .await + self.crate_redirect(req, route_match.params().clone()).await } (&Method::GET, Route::Static(file)) => Ok(App::static_file(*file)), @@ -135,12 +131,11 @@ impl App { let diff = end - start; match &res { - Ok(res) => info!( - logger2, ""; - "status" => res.status().to_string(), - "time" => format!("{}ms", diff.as_millis()) + Ok(res) => tracing::info!( + status = %res.status(), + time = %format_args!("{}ms", diff.as_millis()), ), - Err(err) => error!(logger2, ""; "error" => err.to_string()), + Err(err) => tracing::error!(%err), }; res @@ -152,7 +147,6 @@ impl App { &self, _req: Request, _params: Params, - logger: Logger, ) -> Result, HyperError> { let engine = self.engine.clone(); @@ -161,7 +155,7 @@ impl App { match popular { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render("Could not retrieve popular items", ""); *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; @@ -177,7 +171,6 @@ impl App { &self, req: Request, params: Params, - logger: Logger, format: StatusFormat, ) -> Result, HyperError> { let server = self.clone(); @@ -192,7 +185,7 @@ impl App { match repo_path_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render( "Could not parse repository path", "Please make sure to provide a valid repository path.", @@ -209,7 +202,7 @@ impl App { match analyze_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let response = App::status_format_analysis( None, format, @@ -236,7 +229,6 @@ impl App { &self, _req: Request, params: Params, - logger: Logger, ) -> Result, HyperError> { let engine = self.engine.clone(); @@ -245,7 +237,7 @@ impl App { match crate_name_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render( "Could not parse crate name", "Please make sure to provide a valid crate name.", @@ -261,7 +253,7 @@ impl App { match release_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render( "Could not fetch crate information", "Please make sure to provide a valid crate name.", @@ -302,7 +294,6 @@ impl App { &self, req: Request, params: Params, - logger: Logger, format: StatusFormat, ) -> Result, HyperError> { let server = self.clone(); @@ -332,7 +323,7 @@ impl App { Ok(Some(latest_rel)) => latest_rel.version.to_string(), Ok(None) => return Ok(not_found()), Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render( "Could not fetch crate information", "Please make sure to provide a valid crate name.", @@ -349,7 +340,7 @@ impl App { match crate_path_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let mut response = views::html::error::render( "Could not parse crate path", "Please make sure to provide a valid crate name and version.", @@ -365,7 +356,7 @@ impl App { match analyze_result { Err(err) => { - error!(logger, "error: {}", err); + tracing::error!(%err); let response = App::status_format_analysis( None, format, diff --git a/src/server/views/badge.rs b/src/server/views/badge.rs index 0ee55c5..beba072 100644 --- a/src/server/views/badge.rs +++ b/src/server/views/badge.rs @@ -1,9 +1,7 @@ use badge::{Badge, BadgeOptions}; -use hyper::header::CONTENT_TYPE; -use hyper::{Body, Response}; +use hyper::{header::CONTENT_TYPE, Body, Response}; -use crate::engine::AnalyzeDependenciesOutcome; -use crate::server::ExtraConfig; +use crate::{engine::AnalyzeDependenciesOutcome, server::ExtraConfig}; pub fn badge( analysis_outcome: Option<&AnalyzeDependenciesOutcome>, diff --git a/src/server/views/html/index.rs b/src/server/views/html/index.rs index a1ccae0..748eca0 100644 --- a/src/server/views/html/index.rs +++ b/src/server/views/html/index.rs @@ -1,10 +1,10 @@ use hyper::{Body, Response}; use maud::{html, Markup}; -use crate::models::crates::CratePath; -use crate::models::repo::Repository; - -use crate::server::assets::STATIC_LINKS_JS_PATH; +use crate::{ + models::{crates::CratePath, repo::Repository}, + server::assets::STATIC_LINKS_JS_PATH, +}; fn link_forms() -> Markup { html! { diff --git a/src/server/views/html/mod.rs b/src/server/views/html/mod.rs index 2f3457b..844eb4a 100644 --- a/src/server/views/html/mod.rs +++ b/src/server/views/html/mod.rs @@ -1,15 +1,13 @@ use std::time::Duration; -use hyper::header::CONTENT_TYPE; -use hyper::{Body, Response}; +use hyper::{header::CONTENT_TYPE, Body, Response}; use maud::{html, Markup, Render, DOCTYPE}; pub mod error; pub mod index; pub mod status; -use crate::server::assets::STATIC_STYLE_CSS_PATH; -use crate::server::SELF_BASE_URL; +use crate::server::{assets::STATIC_STYLE_CSS_PATH, SELF_BASE_URL}; fn render_html(title: &str, body: B) -> Response { let rendered = html! { diff --git a/src/server/views/html/status.rs b/src/server/views/html/status.rs index 19c0d23..8017e94 100644 --- a/src/server/views/html/status.rs +++ b/src/server/views/html/status.rs @@ -6,12 +6,15 @@ use pulldown_cmark::{html, Parser}; use rustsec::advisory::Advisory; use semver::Version; -use crate::engine::AnalyzeDependenciesOutcome; -use crate::models::crates::{AnalyzedDependencies, AnalyzedDependency, CrateName}; -use crate::models::repo::RepoSite; -use crate::models::SubjectPath; -use crate::server::views::badge; -use crate::server::ExtraConfig; +use crate::{ + engine::AnalyzeDependenciesOutcome, + models::{ + crates::{AnalyzedDependencies, AnalyzedDependency, CrateName}, + repo::RepoSite, + SubjectPath, + }, + server::{views::badge, ExtraConfig}, +}; fn get_crates_url(name: impl AsRef) -> String { format!("https://crates.io/crates/{}", name.as_ref()) diff --git a/src/utils/cache.rs b/src/utils/cache.rs index 5637472..a09c706 100644 --- a/src/utils/cache.rs +++ b/src/utils/cache.rs @@ -3,7 +3,6 @@ use std::{fmt, sync::Arc, time::Duration}; use derive_more::{Display, Error, From}; use hyper::service::Service; use lru_time_cache::LruCache; -use slog::{debug, Logger}; use tokio::sync::Mutex; #[derive(Debug, Clone, Display, From, Error)] @@ -18,7 +17,6 @@ where { inner: S, cache: Arc>>, - logger: Logger, } impl fmt::Debug for Cache @@ -38,13 +36,12 @@ where S::Response: Clone, Req: Clone + Eq + Ord + fmt::Debug, { - pub fn new(service: S, ttl: Duration, capacity: usize, logger: Logger) -> Cache { + pub fn new(service: S, ttl: Duration, capacity: usize) -> Cache { let cache = LruCache::with_expiry_duration_and_capacity(ttl, capacity); Cache { inner: service, cache: Arc::new(Mutex::new(cache)), - logger, } } @@ -53,19 +50,19 @@ where let mut cache = self.cache.lock().await; if let Some(cached_response) = cache.get(&req) { - debug!( - self.logger, "cache hit"; - "svc" => format!("{:?}", self.inner), - "req" => format!("{:?}", &req) + tracing::debug!( + svc = ?self.inner, + req = ?req, + cache = "hit", ); return Ok(cached_response.clone()); } } - debug!( - self.logger, "cache miss"; - "svc" => format!("{:?}", self.inner), - "req" => format!("{:?}", &req) + tracing::debug!( + svc = ?self.inner, + req = ?req, + cache = "miss", ); let mut service = self.inner.clone(); diff --git a/src/utils/index.rs b/src/utils/index.rs index d2f2587..4e58ae3 100644 --- a/src/utils/index.rs +++ b/src/utils/index.rs @@ -1,32 +1,30 @@ -use std::sync::Arc; -use std::sync::Mutex; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; -use crate::models::crates::CrateName; use anyhow::Result; -use crates_index::Crate; -use crates_index::GitIndex; -use slog::{error, Logger}; -use tokio::task::spawn_blocking; -use tokio::time::{self, MissedTickBehavior}; +use crates_index::{Crate, GitIndex}; +use parking_lot::Mutex; +use tokio::{ + task::spawn_blocking, + time::{self, MissedTickBehavior}, +}; + +use crate::models::crates::CrateName; #[derive(Clone)] pub struct ManagedIndex { index: Arc>, - logger: Logger, } impl ManagedIndex { - pub fn new(logger: Logger) -> Self { + pub fn new() -> Self { // the index path is configurable through the `CARGO_HOME` env variable let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap())); - Self { index, logger } + + Self { index } } pub fn crate_(&self, crate_name: CrateName) -> Option { - let index = self.index.lock().unwrap(); - - index.crate_(crate_name.as_ref()) + self.index.lock().crate_(crate_name.as_ref()) } pub async fn refresh_at_interval(&self, update_interval: Duration) { @@ -34,25 +32,23 @@ impl ManagedIndex { update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); loop { - if let Err(e) = self.refresh().await { - error!( - self.logger, - "failed refreshing the crates.io-index, the operation will be retried: {}", e + if let Err(err) = self.refresh().await { + tracing::error!( + "failed refreshing the crates.io-index, the operation will be retried: {}", + error_reporter::Report::new(err), ); } update_interval.tick().await; } } - async fn refresh(&self) -> Result<()> { + async fn refresh(&self) -> Result<(), crates_index::Error> { let index = Arc::clone(&self.index); - spawn_blocking(move || { - let mut index = index.lock().unwrap(); + spawn_blocking(move || index.lock().update()) + .await + .expect("blocking index update task should never panic")?; - index.update() - }) - .await??; Ok(()) } } From 9a1f07d355d07d6a642b943a26d526ce69e69372 Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Mon, 27 May 2024 00:29:46 +0100 Subject: [PATCH 2/2] refactor: use sparse crates index --- Cargo.lock | 118 ++------------------------------------ Cargo.toml | 14 +++-- src/interactors/crates.rs | 7 +-- src/main.rs | 14 ++--- src/models/crates.rs | 6 ++ src/utils/index.rs | 86 ++++++++++++++++----------- 6 files changed, 78 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66a9d16..4f1ba2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,9 +257,9 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8467f424ab6d70e5daf384289fa68247787effc13901dd0ca46bc9b4a62f1474" dependencies = [ - "gix", "hex", "home", + "http 1.1.0", "memchr", "rustc-hash", "semver", @@ -483,21 +483,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "932dcfbd51320af5f27f1ba02d2e567dec332cac7d2c221ba45d8e767264c4dc" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1600,22 +1585,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.3.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.4" @@ -1839,24 +1808,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1907,50 +1858,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl" -version = "0.10.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" -dependencies = [ - "bitflags 2.5.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "overload" version = "0.1.1" @@ -2069,12 +1982,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - [[package]] name = "platforms" version = "3.4.0" @@ -2281,13 +2188,11 @@ dependencies = [ "http-body-util", "hyper 1.3.1", "hyper-rustls", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2301,7 +2206,6 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower-service", @@ -2612,8 +2516,9 @@ dependencies = [ "error_reporter", "font-awesome-as-a-crate", "futures-util", - "gix", "grass", + "http 1.1.0", + "http-body-util", "hyper 0.14.28", "indexmap", "lru_time_cache", @@ -2629,6 +2534,7 @@ dependencies = [ "serde", "serde_urlencoded", "sha-1", + "static_assertions", "tokio", "toml 0.8.13", "tracing", @@ -2885,16 +2791,6 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.25.0" @@ -3178,12 +3074,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 607ec65..c077da1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,13 +16,15 @@ badge = { path = "./libs/badge" } anyhow = "1" cadence = "1" -crates-index = { version = "2", default-features = false, features = ["git"] } +crates-index = "2" derive_more = "0.99" dotenvy = "0.15" +error_reporter = "1" font-awesome-as-a-crate = "0.3" futures-util = { version = "0.3", default-features = false, features = ["std"] } +http = "1" +http-body-util = "0.1" hyper = { version = "0.14.10", features = ["full"] } -error_reporter = "1" indexmap = { version = "2", features = ["serde"] } lru_time_cache = "0.11" maud = "0.26" @@ -30,7 +32,7 @@ once_cell = "1" parking_lot = "0.12" pulldown-cmark = "0.11" relative-path = { version = "1", features = ["serde"] } -reqwest = { version = "0.12", features = ["json"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "http2", "rustls-tls", "gzip"] } route-recognizer = "0.3" rustsec = "0.29" semver = { version = "1.0", features = ["serde"] } @@ -41,9 +43,9 @@ toml = "0.8" tracing = "0.1.30" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -[target.'cfg(any())'.dependencies] -gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] } - [build-dependencies] grass = { version = "0.13", default-features = false } sha-1 = "0.10" + +[dev-dependencies] +static_assertions = "1" diff --git a/src/interactors/crates.rs b/src/interactors/crates.rs index 1a07f5c..1d81fe2 100644 --- a/src/interactors/crates.rs +++ b/src/interactors/crates.rs @@ -9,7 +9,6 @@ use futures_util::FutureExt as _; use hyper::service::Service; use semver::{Version, VersionReq}; use serde::Deserialize; -use tokio::task::spawn_blocking; use crate::{ models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease}, @@ -68,10 +67,10 @@ impl QueryCrate { index: ManagedIndex, crate_name: CrateName, ) -> anyhow::Result { - let crate_name2 = crate_name.clone(); - let krate = spawn_blocking(move || index.crate_(crate_name2)) + let krate = index + .crate_(&crate_name) .await? - .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?; + .ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_str()))?; convert_pkgs(krate) } diff --git a/src/main.rs b/src/main.rs index c5414ac..799c27d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,15 +79,13 @@ async fn main() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); - let index = ManagedIndex::new(); + let index = ManagedIndex::new(client.clone()); - { - let index = index.clone(); - - tokio::spawn(async move { - index.refresh_at_interval(Duration::from_secs(20)).await; - }); - } + // crates index health check + index + .crate_(&"libc".parse().unwrap()) + .await + .expect("crates index startup check should succeed"); let mut engine = Engine::new(client.clone(), index); engine.set_metrics(metrics); diff --git a/src/models/crates.rs b/src/models/crates.rs index 894ae40..0593cd0 100644 --- a/src/models/crates.rs +++ b/src/models/crates.rs @@ -24,6 +24,12 @@ impl CratePath { #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct CrateName(String); +impl CrateName { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + impl From for String { fn from(crate_name: CrateName) -> String { crate_name.0 diff --git a/src/utils/index.rs b/src/utils/index.rs index 4e58ae3..6838772 100644 --- a/src/utils/index.rs +++ b/src/utils/index.rs @@ -1,54 +1,70 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use anyhow::Result; -use crates_index::{Crate, GitIndex}; -use parking_lot::Mutex; -use tokio::{ - task::spawn_blocking, - time::{self, MissedTickBehavior}, -}; +use crates_index::{Crate, SparseIndex}; use crate::models::crates::CrateName; #[derive(Clone)] pub struct ManagedIndex { - index: Arc>, + index: Arc, + client: reqwest::Client, } impl ManagedIndex { - pub fn new() -> Self { + pub fn new(client: reqwest::Client) -> Self { // the index path is configurable through the `CARGO_HOME` env variable - let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap())); + let index = Arc::new(SparseIndex::new_cargo_default().unwrap()); - Self { index } + Self { index, client } } - pub fn crate_(&self, crate_name: CrateName) -> Option { - self.index.lock().crate_(crate_name.as_ref()) - } + /// Finds crate by name, returning a list of its versions. + /// + /// # Errors + /// + /// Returns error if HTTP request to crates index fails. + pub async fn crate_(&self, crate_name: &CrateName) -> anyhow::Result> { + let req = self.index.make_cache_request(crate_name.as_str())?; + let req = http_to_reqwest_req(req); + + let res = self.client.execute(req).await?; + let res = reqwest_to_http_res(res).await?; - pub async fn refresh_at_interval(&self, update_interval: Duration) { - let mut update_interval = time::interval(update_interval); - update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); - - loop { - if let Err(err) = self.refresh().await { - tracing::error!( - "failed refreshing the crates.io-index, the operation will be retried: {}", - error_reporter::Report::new(err), - ); - } - update_interval.tick().await; - } + Ok(self + .index + .parse_cache_response(crate_name.as_str(), res, true)?) } +} - async fn refresh(&self) -> Result<(), crates_index::Error> { - let index = Arc::clone(&self.index); +/// Converts an `http` request builder from `crates-index` to a `reqwest` request. +fn http_to_reqwest_req(req: http::request::Builder) -> reqwest::Request { + let req = req + .body(()) + .expect("request from crates_index crate should be valid"); - spawn_blocking(move || index.lock().update()) - .await - .expect("blocking index update task should never panic")?; + let req = http::Request::from_parts(req.into_parts().0, Vec::new()); - Ok(()) - } + reqwest::Request::try_from(req).expect("request from crates_index crate should be valid") +} + +/// Converts an `http` request builder from `crates-index` to a `reqwest` request. +/// +/// # Errors +/// +/// Returns error if reading HTTP response fails (e.g.: connection is lost while streaming payload). +async fn reqwest_to_http_res(res: reqwest::Response) -> anyhow::Result>> { + use http_body_util::BodyExt as _; + + let (res, body) = http::Response::from(res).into_parts(); + + let body = body.collect().await?.to_bytes().to_vec(); + + Ok(http::Response::from_parts(res, body)) +} + +#[cfg(test)] +mod tests { + use super::*; + + static_assertions::assert_impl_all!(ManagedIndex: Send, Sync); }