Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update tera/minifier & cut some memory allocations #719

Merged
merged 4 commits into from
Aug 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
238 changes: 157 additions & 81 deletions Cargo.lock

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
name = "crater"
version = "0.1.0"
edition = "2021"

build = "build.rs"
default-run = "crater"

[profile.dev]
opt-level = 0
Expand All @@ -24,7 +24,7 @@ http = "0.2"
hyper = "0.14"
lazy_static = "1.0"
mime = "0.3.1"
minifier = { version = "0.2", features = ["html"] }
minifier = { version = "0.3", features = ["html"] }
r2d2 = "0.8.2"
r2d2_sqlite = "0.22.0"
rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] }
Expand All @@ -38,7 +38,7 @@ serde_regex = "1.1.0"
clap = { version = "4", features = ["derive"] }
tar = "0.4.36"
tempfile = "3.0.0"
tera = "0.11.7"
tera = "1.19.1"
toml = "0.8.6"
url = "2"
walkdir = "2"
Expand All @@ -60,7 +60,7 @@ tokio = "1.24"
aws-sdk-s3 = "1.7"
aws-config = { version = "1", features = ["behavior-version-latest"] }
thiserror = "1.0.38"
nix = { version = "0.27.1", features = ["mman"] }
nix = { version = "0.27.1", features = ["mman", "resource"] }

[dev-dependencies]
assert_cmd = "2.0.4"
Expand Down
5 changes: 3 additions & 2 deletions src/assets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ fn build_tera_cache() -> Fallible<Tera> {
}

#[allow(unused_variables)]
pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String> {
pub fn render_template<C: Serialize>(name: &str, context: C) -> Fallible<String> {
// On debug builds the cache is rebuilt every time to pick up changed templates
let tera_owned: Tera;
let tera;
Expand All @@ -161,7 +161,8 @@ pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String
tera = &TERA_CACHE;
}

let tera_context = tera::Context::from_serialize(context)?;
Ok(tera
.render(name, context)
.render(name, &tera_context)
.map_err(|e| failure::format_err!("{:?}", e))?)
}
84 changes: 84 additions & 0 deletions src/bin/test-report.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
use crater::experiments::ExperimentDBRecord;
use crater::report::ReportWriter;
use crater::results::EncodingType;
use crater::{config::Config, db::QueryUtils};
use failure::Fallible;
use mime::{self, Mime};
use std::{borrow::Cow, fmt, path::Path};

#[cfg(not(unix))]
fn main() {
eprintln!("Not implemented!");
}

#[cfg(unix)]
fn main() {
let mut env = env_logger::Builder::new();
env.filter_module("test_report", log::LevelFilter::Info);
env.filter_module("crater", log::LevelFilter::Info);
env.filter_module("rustwide", log::LevelFilter::Info);
if let Ok(content) = std::env::var("RUST_LOG") {
env.parse_filters(&content);
}
rustwide::logging::init_with(env.build());
let config: Config = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap();
let db = crater::db::Database::open_at(std::path::Path::new("crater.db")).unwrap();
let experiments = db
.query("SELECT * FROM experiments;", [], |r| {
ExperimentDBRecord::from_row(r)
})
.unwrap();
let experiments: Vec<_> = experiments
.into_iter()
.map(|record| record.into_experiment())
.collect::<Fallible<_>>()
.unwrap();
let ex = experiments.iter().find(|e| e.name == "pr-118920").unwrap();
let rdb = crater::results::DatabaseDB::new(&db);

log::info!("Getting crates...");

let crates = ex.get_crates(&db).unwrap();
let writer = NullWriter;

log::info!("Starting report generation...");
log::info!(
"@ {:?}",
nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
.unwrap()
.max_rss()
);
crater::report::gen(&rdb, ex, &crates, &writer, &config, false).unwrap();
log::info!(
"@ {:?}",
nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
.unwrap()
.max_rss()
);
}

#[derive(Debug)]
struct NullWriter;

impl ReportWriter for NullWriter {
fn write_bytes<P: AsRef<Path>>(
&self,
_path: P,
_b: &[u8],
_mime: &Mime,
_encoding_type: EncodingType,
) -> Fallible<()> {
// no-op
Ok(())
}
fn write_string<P: AsRef<Path>>(&self, _path: P, _s: Cow<str>, _mime: &Mime) -> Fallible<()> {
// no-op
Ok(())
}
}

impl fmt::Display for NullWriter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
24 changes: 24 additions & 0 deletions src/db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use r2d2::{CustomizeConnection, Pool};
use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::types::ToSql;
use rusqlite::{Connection, Row, Transaction};
use std::path::Path;
use std::sync::Arc;
use std::time::Instant;
use tempfile::NamedTempFile;
Expand Down Expand Up @@ -55,6 +56,11 @@ impl Database {
Database::new(SqliteConnectionManager::file(path), None)
}

pub fn open_at(path: &Path) -> Fallible<Self> {
std::fs::create_dir_all(&*WORK_DIR)?;
Database::new(SqliteConnectionManager::file(path), None)
}

#[cfg(test)]
pub fn temp() -> Fallible<Self> {
let tempfile = NamedTempFile::new()?;
Expand Down Expand Up @@ -191,6 +197,24 @@ pub trait QueryUtils {
})
}

fn query_row<T, F: FnOnce(&Row) -> Fallible<T>>(
&self,
sql: &str,
params: impl rusqlite::Params,
func: F,
) -> Fallible<Option<T>> {
self.with_conn(|conn| {
self.trace(sql, || {
let mut prepared = conn.prepare(sql)?;
let mut rows = prepared.query(params)?;
if let Ok(Some(row)) = rows.next() {
return Ok(Some(func(row)?));
}
Ok(None)
})
})
}

fn trace<T, F: FnOnce() -> T>(&self, sql: &str, f: F) -> T {
let start = Instant::now();
let res = f();
Expand Down
6 changes: 3 additions & 3 deletions src/experiments.rs
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,7 @@ impl Experiment {
}
}

struct ExperimentDBRecord {
pub struct ExperimentDBRecord {
name: String,
mode: String,
cap_lints: String,
Expand All @@ -680,7 +680,7 @@ struct ExperimentDBRecord {
}

impl ExperimentDBRecord {
fn from_row(row: &Row) -> rusqlite::Result<Self> {
pub fn from_row(row: &Row) -> rusqlite::Result<Self> {
Ok(ExperimentDBRecord {
name: row.get("name")?,
mode: row.get("mode")?,
Expand All @@ -702,7 +702,7 @@ impl ExperimentDBRecord {
})
}

fn into_experiment(self) -> Fallible<Experiment> {
pub fn into_experiment(self) -> Fallible<Experiment> {
Ok(Experiment {
name: self.name,
toolchains: [self.toolchain_start.parse()?, self.toolchain_end.parse()?],
Expand Down
9 changes: 5 additions & 4 deletions src/report/html.rs
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,6 @@ fn write_report<W: ReportWriter>(
};

info!("generating {}", to);
let rendered = assets::render_template("report/results.html", &context)
.context("rendering template report/results.html")?;
let html = minifier::html::minify(&rendered);
dest.write_string(to, html.into(), &mime::TEXT_HTML)?;

if output_templates {
dest.write_string(
Expand All @@ -269,6 +265,11 @@ fn write_report<W: ReportWriter>(
)?;
}

let rendered = assets::render_template("report/results.html", context)
.context("rendering template report/results.html")?;
let html = minifier::html::minify(&rendered);
dest.write_string(to, html.into(), &mime::TEXT_HTML)?;

Ok(())
}

Expand Down
19 changes: 5 additions & 14 deletions src/results/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,22 +179,13 @@ impl<'a> ReadResults for DatabaseDB<'a> {
toolchain: &Toolchain,
krate: &Crate,
) -> Fallible<Option<TestResult>> {
let result: Option<String> = self
.db
.query(
"SELECT result FROM results \
Ok(self.db.query_row(
"SELECT result FROM results \
WHERE experiment = ?1 AND toolchain = ?2 AND crate = ?3 \
LIMIT 1;",
[&ex.name, &toolchain.to_string(), &krate.id()],
|row| row.get("result"),
)?
.pop();

if let Some(res) = result {
Ok(Some(res.parse()?))
} else {
Ok(None)
}
[&ex.name, &toolchain.to_string(), &krate.id()],
|row| Ok(row.get_ref("result")?.as_str()?.parse::<TestResult>()?),
)?)
}
}

Expand Down
Loading