From 526efa74ea97660bc6be1bb4f37094208ef4b26f Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Fri, 22 Dec 2023 10:48:45 -0500 Subject: [PATCH] Move to latest tera/minifier Also adds a test binary for running report generation locally. --- Cargo.lock | 235 +++++++++++++++++++++++++++-------------- Cargo.toml | 6 +- src/assets.rs | 2 +- src/bin/test-report.rs | 78 ++++++++++++++ src/db/mod.rs | 6 ++ src/experiments.rs | 6 +- 6 files changed, 246 insertions(+), 87 deletions(-) create mode 100644 src/bin/test-report.rs diff --git a/Cargo.lock b/Cargo.lock index d92f49e8..0e5c4447 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,7 +145,7 @@ dependencies = [ "log", "native-tls", "openssl", - "url 2.4.1", + "url", "wildmatch", ] @@ -231,7 +231,7 @@ dependencies = [ "aws-types", "fastrand", "http", - "percent-encoding 2.3.0", + "percent-encoding", "tracing", "uuid", ] @@ -260,10 +260,10 @@ dependencies = [ "http", "http-body", "once_cell", - "percent-encoding 2.3.0", + "percent-encoding", "regex", "tracing", - "url 2.4.1", + "url", ] [[package]] @@ -352,7 +352,7 @@ dependencies = [ "http", "once_cell", "p256", - "percent-encoding 2.3.0", + "percent-encoding", "regex", "ring", "sha2", @@ -420,7 +420,7 @@ dependencies = [ "http", "http-body", "once_cell", - "percent-encoding 2.3.0", + "percent-encoding", "pin-project-lite", "pin-utils", "tracing", @@ -726,6 +726,28 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "chrono-tz" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clap" version = "4.4.7" @@ -849,7 +871,7 @@ dependencies = [ "mime", "minifier", "nix 0.27.1", - "percent-encoding 2.3.0", + "percent-encoding", "predicates", "prometheus", "r2d2", @@ -873,7 +895,7 @@ dependencies = [ "thiserror", "tokio", "toml 0.8.6", - "url 2.4.1", + "url", "walkdir", "warp", "zstd", @@ -1244,16 +1266,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "backtrace", - "version_check", -] - [[package]] name = "failure" version = "0.1.8" @@ -1372,7 +1384,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ - "percent-encoding 2.3.0", + "percent-encoding", ] [[package]] @@ -1486,7 +1498,7 @@ dependencies = [ "log", "openssl-probe", "openssl-sys", - "url 2.4.1", + "url", ] [[package]] @@ -2169,7 +2181,7 @@ dependencies = [ "gix-path", "home", "thiserror", - "url 2.4.1", + "url", ] [[package]] @@ -2210,10 +2222,28 @@ dependencies = [ ] [[package]] -name = "glob" -version = "0.2.11" +name = "globset" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +dependencies = [ + "bitflags 1.3.2", + "ignore", + "walkdir", +] [[package]] name = "group" @@ -2369,9 +2399,12 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humansize" -version = "1.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] [[package]] name = "humantime" @@ -2457,23 +2490,28 @@ dependencies = [ [[package]] name = "idna" -version = "0.1.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ - "matches", "unicode-bidi", "unicode-normalization", ] [[package]] -name = "idna" -version = "0.4.0" +name = "ignore" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", ] [[package]] @@ -2584,6 +2622,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libsqlite3-sys" version = "0.26.0" @@ -2653,12 +2697,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "maybe-async" version = "0.2.7" @@ -2731,9 +2769,9 @@ dependencies = [ [[package]] name = "minifier" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5394aa376422b4b2b6c02fd9cfcb657e4ec544ae98e43d7d5d785fd0d042fd6d" +checksum = "95bbbf96b9ac3482c2a25450b67a15ed851319bc5fabf3b40742ea9066e84282" dependencies = [ "regex", ] @@ -2979,10 +3017,13 @@ dependencies = [ ] [[package]] -name = "percent-encoding" -version = "1.0.1" +name = "parse-zoneinfo" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] [[package]] name = "percent-encoding" @@ -3035,6 +3076,44 @@ dependencies = [ "sha2", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -3325,7 +3404,7 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding 2.3.0", + "percent-encoding", "pin-project-lite", "serde", "serde_json", @@ -3334,7 +3413,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tower-service", - "url 2.4.1", + "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -3485,7 +3564,7 @@ dependencies = [ "lazy_static", "log", "nix 0.25.1", - "percent-encoding 2.3.0", + "percent-encoding", "remove_dir_all", "scopeguard", "serde", @@ -3731,6 +3810,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -3907,23 +3992,24 @@ dependencies = [ [[package]] name = "tera" -version = "0.11.20" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b505279e19d8f7d24b1a9dc58327c9c36174b1a2c7ebdeac70792d017cb64f3" +checksum = "970dff17c11e884a4a09bc76e3a17ef71e01bb13447a11e85226e254fe6d10b8" dependencies = [ "chrono", - "error-chain", - "glob", + "chrono-tz", + "globwalk", "humansize", "lazy_static", + "percent-encoding", "pest", "pest_derive", + "rand", "regex", "serde", "serde_json", "slug", "unic-segment", - "url 1.7.2", ] [[package]] @@ -4183,7 +4269,7 @@ dependencies = [ "rand", "sha1", "thiserror", - "url 2.4.1", + "url", "utf-8", ] @@ -4210,39 +4296,39 @@ dependencies = [ [[package]] name = "unic-char-property" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36d3f7ce754afdbccccf8ff0dd0134e50fb44aaae579f96218856e9e5dbd1e" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" dependencies = [ "unic-char-range", ] [[package]] name = "unic-char-range" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ab85fab42ad1b26cafc03bf891f69cb4d6e15f491030e89a0122197baa8ae8" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" [[package]] name = "unic-common" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8d4a7ade929ef7d971e16ced21a8cd56a63869aa6032dfb8cb083cf7d077bf" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" [[package]] name = "unic-segment" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ca47cbb09fb5fcd066b5867d11dc528302fa465277882797d6a836e1ee6f9e" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" dependencies = [ "unic-ucd-segment", ] [[package]] name = "unic-ucd-segment" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f1a08ce0409a9e391b88d1930118eec48af12742fc538bcec55f775865776e" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" dependencies = [ "unic-char-property", "unic-char-range", @@ -4251,9 +4337,9 @@ dependencies = [ [[package]] name = "unic-ucd-version" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1f5e6c6c53c2d0ece4a5964bc55fcff8602153063cb4fab20958ff32998ff6" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" dependencies = [ "unic-common", ] @@ -4306,17 +4392,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - [[package]] name = "url" version = "2.4.1" @@ -4324,8 +4399,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", - "idna 0.4.0", - "percent-encoding 2.3.0", + "idna", + "percent-encoding", ] [[package]] @@ -4418,7 +4493,7 @@ dependencies = [ "mime", "mime_guess", "multer", - "percent-encoding 2.3.0", + "percent-encoding", "pin-project", "rustls-pemfile", "scoped-tls", diff --git a/Cargo.toml b/Cargo.toml index e5157d5c..55e89b40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ http = "0.2" hyper = "0.14" lazy_static = "1.0" mime = "0.3.1" -minifier = { version = "0.2", features = ["html"] } +minifier = { version = "0.3", features = ["html"] } r2d2 = "0.8.2" r2d2_sqlite = "0.22.0" rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] } @@ -38,7 +38,7 @@ serde_regex = "1.1.0" clap = { version = "4", features = ["derive"] } tar = "0.4.36" tempfile = "3.0.0" -tera = "0.11.7" +tera = "1.19.1" toml = "0.8.6" url = "2" walkdir = "2" @@ -60,7 +60,7 @@ tokio = "1.24" aws-sdk-s3 = "1.7" aws-config = { version = "1", features = ["behavior-version-latest"] } thiserror = "1.0.38" -nix = { version = "0.27.1", features = ["mman"] } +nix = { version = "0.27.1", features = ["mman", "resource"] } [dev-dependencies] assert_cmd = "2.0.4" diff --git a/src/assets.rs b/src/assets.rs index c2ba848e..f8294776 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -162,6 +162,6 @@ pub fn render_template(name: &str, context: &C) -> Fallible = experiments + .into_iter() + .map(|record| record.into_experiment()) + .collect::>() + .unwrap(); + let ex = experiments.iter().find(|e| e.name == "pr-118920").unwrap(); + let rdb = crater::results::DatabaseDB::new(&db); + + log::info!("Getting crates..."); + + let crates = ex.get_crates(&db).unwrap(); + let writer = NullWriter; + + log::info!("Starting report generation..."); + log::info!( + "@ {:?}", + nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF) + .unwrap() + .max_rss() + ); + crater::report::gen(&rdb, &ex, &crates, &writer, &config, false).unwrap(); + log::info!( + "@ {:?}", + nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF) + .unwrap() + .max_rss() + ); +} + +#[derive(Debug)] +struct NullWriter; + +impl ReportWriter for NullWriter { + fn write_bytes>( + &self, + _path: P, + _b: &[u8], + _mime: &Mime, + _encoding_type: EncodingType, + ) -> Fallible<()> { + // no-op + Ok(()) + } + fn write_string>(&self, _path: P, _s: Cow, _mime: &Mime) -> Fallible<()> { + // no-op + Ok(()) + } +} + +impl fmt::Display for NullWriter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 154f2a80..6d8ed85e 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -6,6 +6,7 @@ use r2d2::{CustomizeConnection, Pool}; use r2d2_sqlite::SqliteConnectionManager; use rusqlite::types::ToSql; use rusqlite::{Connection, Row, Transaction}; +use std::path::Path; use std::sync::Arc; use std::time::Instant; use tempfile::NamedTempFile; @@ -55,6 +56,11 @@ impl Database { Database::new(SqliteConnectionManager::file(path), None) } + pub fn open_at(path: &Path) -> Fallible { + std::fs::create_dir_all(&*WORK_DIR)?; + Database::new(SqliteConnectionManager::file(path), None) + } + #[cfg(test)] pub fn temp() -> Fallible { let tempfile = NamedTempFile::new()?; diff --git a/src/experiments.rs b/src/experiments.rs index 9e57e3e4..5374b989 100644 --- a/src/experiments.rs +++ b/src/experiments.rs @@ -659,7 +659,7 @@ impl Experiment { } } -struct ExperimentDBRecord { +pub struct ExperimentDBRecord { name: String, mode: String, cap_lints: String, @@ -680,7 +680,7 @@ struct ExperimentDBRecord { } impl ExperimentDBRecord { - fn from_row(row: &Row) -> rusqlite::Result { + pub fn from_row(row: &Row) -> rusqlite::Result { Ok(ExperimentDBRecord { name: row.get("name")?, mode: row.get("mode")?, @@ -702,7 +702,7 @@ impl ExperimentDBRecord { }) } - fn into_experiment(self) -> Fallible { + pub fn into_experiment(self) -> Fallible { Ok(Experiment { name: self.name, toolchains: [self.toolchain_start.parse()?, self.toolchain_end.parse()?],