From d9463199255ddc46f82f8d03a1bf38de390096a3 Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Sun, 9 Jan 2022 22:57:30 -0500 Subject: [PATCH] Add a CPU limit, raise example memory limit The combination of no job limit and a low memory limit has been causing a lot of OOMs in crater runs. This adds a job limit and enforces it including for doctests, which are built by the test runner. If nothing else, setting this CPU limit provides reproducible OOMs because many crates have peak memory usage proportional to build parallelism. --- Cargo.lock | 9 ++-- Cargo.toml | 1 + config.toml | 3 +- src/config.rs | 3 ++ src/runner/test.rs | 42 +++++++++++++++++-- tests/check_config/bad-duplicate-crate.toml | 1 + tests/check_config/bad-missing-crate.toml | 1 + tests/check_config/bad-missing-repo.toml | 1 + tests/check_config/good.toml | 1 + tests/minicrater/blacklist/config.toml | 1 + tests/minicrater/clippy/config.toml | 1 + tests/minicrater/full/config.toml | 1 + tests/minicrater/ignore-blacklist/config.toml | 1 + tests/minicrater/missing-repo/config.toml | 1 + .../resource-exhaustion/config.toml | 1 + tests/minicrater/small/config.toml | 1 + 16 files changed, 61 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9749049d1..1da72242a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,6 +428,7 @@ dependencies = [ "env_logger", "failure", "flate2", + "git2", "hmac 0.7.1", "http 0.1.21", "hyper 0.12.35", @@ -1028,9 +1029,9 @@ checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" [[package]] name = "git2" -version = "0.13.12" +version = "0.13.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6f1a0238d7f8f8fd5ee642f4ebac4dbc03e03d1f78fbe7a3ede35dcf7e2224" +checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" dependencies = [ "bitflags", "libc", @@ -1481,9 +1482,9 @@ checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "libgit2-sys" -version = "0.12.14+1.1.0" +version = "0.12.26+1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f25af58e6495f7caf2919d08f212de550cfa3ed2f5e744988938ea292b9f549" +checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 822657cc5..1990754b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ ctrlc = "3.1.3" prometheus = "0.7.0" cargo_metadata = "0.12.1" indexmap = "1.4.0" +git2 = { version = "0.13", features = ["vendored-libgit2"] } [dev-dependencies] assert_cmd = "1.0.2" diff --git a/config.toml b/config.toml index df959f59d..06475de29 100644 --- a/config.toml +++ b/config.toml @@ -27,7 +27,8 @@ local-crates = [] [sandbox] # Maximum amount of RAM allowed during builds -memory-limit = "1536M" # 1.5G +memory-limit = "4096M" # 4G, 1G/cpu +cpu-limit = 4 # Restrictions on the amount of information stored in build logs build-log-max-size = "5M" build-log-max-lines = 10000 diff --git a/src/config.rs b/src/config.rs index ae637837b..d36d085b2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -74,6 +74,7 @@ pub struct DemoCrates { #[serde(rename_all = "kebab-case")] pub struct SandboxConfig { pub memory_limit: Size, + pub cpu_limit: u16, pub build_log_max_size: Size, pub build_log_max_lines: usize, } @@ -256,6 +257,7 @@ impl Default for Config { local_crates: HashMap::new(), sandbox: SandboxConfig { memory_limit: Size::Gigabytes(2), + cpu_limit: 1, build_log_max_size: Size::Megabytes(1), build_log_max_lines: 1000, }, @@ -308,6 +310,7 @@ mod tests { "local-crates = []\n", "[sandbox]\n", "memory-limit = \"2G\"\n", + "cpu-limit = 1\n", "build-log-max-size = \"2M\"\n", "build-log-max-lines = 1000\n", "[crates]\n", diff --git a/src/runner/test.rs b/src/runner/test.rs index f1473095d..de31e914e 100644 --- a/src/runner/test.rs +++ b/src/runner/test.rs @@ -89,6 +89,9 @@ fn run_cargo( rustflags.push(' '); rustflags.push_str(tc_rustflags); } + // Limit debuginfo in an effort to decrease linker peak memory usage, while retaining enough + // information to get useful backtraces. + rustflags.push_str(" -Cdebuginfo=1"); let rustflags_env = if let Some(&"doc") = args.get(0) { "RUSTDOCFLAGS" @@ -210,6 +213,7 @@ pub(super) fn run_test( ); let sandbox = SandboxBuilder::new() .memory_limit(Some(ctx.config.sandbox.memory_limit.to_bytes())) + .cpu_limit(Some(f32::from(ctx.config.sandbox.cpu_limit))) .enable_networking(false); let krate = &ctx.krate.to_rustwide(); @@ -235,17 +239,31 @@ fn build( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible<()> { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); run_cargo( ctx, build_env, - &["build", "--frozen", "--message-format=json"], + &[ + "build", + "--frozen", + "--message-format=json", + "--jobs", + &cpus, + ], true, local_packages_id, )?; run_cargo( ctx, build_env, - &["test", "--frozen", "--no-run", "--message-format=json"], + &[ + "test", + "--frozen", + "--no-run", + "--message-format=json", + "--jobs", + &cpus, + ], true, local_packages_id, )?; @@ -253,10 +271,19 @@ fn build( } fn test(ctx: &TaskCtx, build_env: &Build) -> Fallible<()> { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); run_cargo( ctx, build_env, - &["test", "--frozen"], + &[ + "test", + "--frozen", + "--jobs", + &cpus, + "--", + "--test-threads", + &cpus, + ], false, &HashSet::new(), ) @@ -299,6 +326,7 @@ pub(super) fn test_check_only( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); if let Err(err) = run_cargo( ctx, build_env, @@ -308,6 +336,8 @@ pub(super) fn test_check_only( "--all", "--all-targets", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, @@ -323,6 +353,7 @@ pub(super) fn test_clippy_only( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); if let Err(err) = run_cargo( ctx, build_env, @@ -332,6 +363,8 @@ pub(super) fn test_clippy_only( "--all", "--all-targets", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, @@ -347,6 +380,7 @@ pub(super) fn test_rustdoc( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); let res = run_cargo( ctx, build_env, @@ -356,6 +390,8 @@ pub(super) fn test_rustdoc( "--no-deps", "--document-private-items", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, diff --git a/tests/check_config/bad-duplicate-crate.toml b/tests/check_config/bad-duplicate-crate.toml index f4e438e0d..b2e553750 100644 --- a/tests/check_config/bad-duplicate-crate.toml +++ b/tests/check_config/bad-duplicate-crate.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/bad-missing-crate.toml b/tests/check_config/bad-missing-crate.toml index 3fff66502..d5d2f6d36 100644 --- a/tests/check_config/bad-missing-crate.toml +++ b/tests/check_config/bad-missing-crate.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/bad-missing-repo.toml b/tests/check_config/bad-missing-repo.toml index 6b077bc3b..7477fee72 100644 --- a/tests/check_config/bad-missing-repo.toml +++ b/tests/check_config/bad-missing-repo.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit= 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/good.toml b/tests/check_config/good.toml index 4f9903b27..1de068867 100644 --- a/tests/check_config/good.toml +++ b/tests/check_config/good.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/blacklist/config.toml b/tests/minicrater/blacklist/config.toml index 7a04174b8..a349ade06 100644 --- a/tests/minicrater/blacklist/config.toml +++ b/tests/minicrater/blacklist/config.toml @@ -18,6 +18,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/clippy/config.toml b/tests/minicrater/clippy/config.toml index c64180af7..6ec5404fd 100644 --- a/tests/minicrater/clippy/config.toml +++ b/tests/minicrater/clippy/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "clippy-warn"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/full/config.toml b/tests/minicrater/full/config.toml index b138311b9..f23f1763b 100644 --- a/tests/minicrater/full/config.toml +++ b/tests/minicrater/full/config.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/ignore-blacklist/config.toml b/tests/minicrater/ignore-blacklist/config.toml index 973f0be1b..2d96b5b90 100644 --- a/tests/minicrater/ignore-blacklist/config.toml +++ b/tests/minicrater/ignore-blacklist/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/missing-repo/config.toml b/tests/minicrater/missing-repo/config.toml index 7f509facc..407f36266 100644 --- a/tests/minicrater/missing-repo/config.toml +++ b/tests/minicrater/missing-repo/config.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/resource-exhaustion/config.toml b/tests/minicrater/resource-exhaustion/config.toml index e74994989..799c127e7 100644 --- a/tests/minicrater/resource-exhaustion/config.toml +++ b/tests/minicrater/resource-exhaustion/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "memory-hungry"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/small/config.toml b/tests/minicrater/small/config.toml index fcedc7cfb..98f5160b4 100644 --- a/tests/minicrater/small/config.toml +++ b/tests/minicrater/small/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "beta-regression"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000