From 827140a4eecd120bd7c8fe0df18a5495131bcd76 Mon Sep 17 00:00:00 2001 From: Li-yao Xia Date: Tue, 3 Dec 2024 12:26:37 +0100 Subject: [PATCH] Condense test logs by hiding successful tests using ANSI codes - Disable ANSI codes when stdout is not a terminal - Add special case for running in CI --- creusot/Cargo.toml | 2 +- creusot/tests/creusot-contracts.rs | 43 +++++++++++--- creusot/tests/diff.rs | 3 +- creusot/tests/ui.rs | 93 ++++++++++++++++++++++-------- 4 files changed, 107 insertions(+), 34 deletions(-) diff --git a/creusot/Cargo.toml b/creusot/Cargo.toml index 3a8afcba3..4f34242a7 100644 --- a/creusot/Cargo.toml +++ b/creusot/Cargo.toml @@ -16,7 +16,6 @@ petgraph = { version = "0.6", git = "https://github.com/xldenis/petgraph", rev = indexmap = { version = "1.7.0", features = ["serde"] } toml = "0.5.8" why3 = { path = "../why3", features = ["serialize"] } -clap = { version = "4.2", features = ["derive", "env"] } creusot-metadata = { path = "../creusot-metadata" } include_dir = "0.7.3" tempdir = "0.3.7" @@ -25,6 +24,7 @@ lazy_static = "1.4.0" pathdiff = "0.2" [dev-dependencies] +clap = { version = "4.2", features = ["derive", "env"] } regex = "1.10.5" glob = "*" assert_cmd = "1.0" diff --git a/creusot/tests/creusot-contracts.rs b/creusot/tests/creusot-contracts.rs index 37b692c39..31be63936 100644 --- a/creusot/tests/creusot-contracts.rs +++ b/creusot/tests/creusot-contracts.rs @@ -1,11 +1,31 @@ -use std::{io::Write as _, path::PathBuf}; +use clap::Parser; +use std::{ + env, + io::{IsTerminal, Write as _}, + path::PathBuf, +}; use termcolor::{BufferWriter, Color, ColorChoice, ColorSpec, StandardStream, WriteColor as _}; mod diff; use diff::differ; +#[derive(Debug, Parser)] +struct Args { + /// Force color output + #[clap(long)] + force_color: bool, + /// Overwrite expected output files with actual output + #[clap(long)] + bless: bool, + /// Only run tests which contain this string + filter: Option, +} + fn main() { - let bless = std::env::args().any(|arg| arg == "--bless"); + let mut args = Args::parse(); + if env::var("CI").is_ok() { + args.force_color = true; + } // Build creusot-rustc to make it available to cargo-creusot let _ = escargot::CargoBuild::new() .bin("creusot-rustc") @@ -43,13 +63,21 @@ fn main() { "creusot-contracts", ]); - let output = cargo_creusot.output().unwrap(); - let mut out = StandardStream::stdout(ColorChoice::Always); + let is_tty = std::io::stdout().is_terminal(); + let mut out = StandardStream::stdout(if args.force_color || is_tty { + ColorChoice::Always + } else { + ColorChoice::Never + }); + + write!(out, "Testing creusot-contracts ... ").unwrap(); + out.flush().unwrap(); + let output = cargo_creusot.output().unwrap(); let stdout = PathBuf::from("tests/creusot-contracts/creusot-contracts.coma"); let mut failed = false; - if bless { + if args.bless { if output.stdout.is_empty() { panic!( "creusot-contracts should have an output! stderr is:\n\n{}", @@ -59,7 +87,7 @@ fn main() { out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap(); writeln!(&mut out, "blessed").unwrap(); out.reset().unwrap(); - let (success, _) = differ(output.clone(), &stdout, None, true).unwrap(); + let (success, _) = differ(output.clone(), &stdout, None, true, is_tty).unwrap(); if !success { out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap(); @@ -69,7 +97,7 @@ fn main() { std::fs::write(stdout, &output.stdout).unwrap(); } else { - let (success, mut buf) = differ(output.clone(), &stdout, None, true).unwrap(); + let (success, buf) = differ(output.clone(), &stdout, None, true, is_tty).unwrap(); if success { out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap(); @@ -82,7 +110,6 @@ fn main() { }; out.reset().unwrap(); - buf.reset().unwrap(); let wrt = BufferWriter::stdout(ColorChoice::Always); wrt.print(&buf).unwrap(); } diff --git a/creusot/tests/diff.rs b/creusot/tests/diff.rs index 78ef7769e..1e533d441 100644 --- a/creusot/tests/diff.rs +++ b/creusot/tests/diff.rs @@ -19,8 +19,9 @@ pub fn differ( stdout: &Path, stderr: Option<&Path>, should_succeed: bool, + enable_color: bool, ) -> Result<(bool, Buffer), Box> { - let mut buf = Buffer::ansi(); + let mut buf = if enable_color { Buffer::ansi() } else { Buffer::no_color() }; match output.clone().ok() { Ok(output) => { let expect_out = &std::fs::read(stdout).unwrap_or_else(|_| Vec::new()); diff --git a/creusot/tests/ui.rs b/creusot/tests/ui.rs index 434963f56..2dba5d77d 100644 --- a/creusot/tests/ui.rs +++ b/creusot/tests/ui.rs @@ -1,7 +1,8 @@ +use clap::Parser; use std::{ env, fs::File, - io::{BufRead, BufReader, Write}, + io::{BufRead, BufReader, IsTerminal, Write}, path::{Path, PathBuf}, process::Command, }; @@ -10,6 +11,21 @@ use termcolor::*; mod diff; use diff::{differ, normalize_file_path}; +#[derive(Debug, Parser)] +struct Args { + /// Suppress all output other than failing test cases + #[clap(long)] + quiet: bool, + /// Force color output + #[clap(long)] + force_color: bool, + /// Overwrite expected output files with actual output + #[clap(long)] + bless: bool, + /// Only run tests which contain this string + filter: Option, +} + fn main() { // Build `creusot-rustc` and `cargo-creusot` @@ -55,12 +71,22 @@ fn main() { std::process::exit(1); } - should_fail("tests/should_fail/**/*.rs", |p| { + let mut args = Args::parse(); + if env::var("CI").is_ok() { + args.quiet = true; + args.force_color = true; + } + + should_fail("tests/should_fail/**/*.rs", &args, |p| { run_creusot(creusot_rustc.path(), p, &temp_file.to_string_lossy()) }); - should_succeed("tests/should_succeed/**/*.rs", |p| { + should_succeed("tests/should_succeed/**/*.rs", &args, |p| { run_creusot(creusot_rustc.path(), p, &temp_file.to_string_lossy()) }); + + if !args.quiet { + println!("All tests passed!"); + } } fn run_creusot( @@ -130,36 +156,39 @@ fn run_creusot( Some(cmd) } -fn should_succeed(s: &str, b: B) +fn should_succeed(s: &str, args: &Args, b: B) where B: Fn(&Path) -> Option, { - glob_runner(s, b, true); + glob_runner(s, args, b, true); } -fn should_fail(s: &str, b: B) +fn should_fail(s: &str, args: &Args, b: B) where B: Fn(&Path) -> Option, { - glob_runner(s, b, false); + glob_runner(s, args, b, false); } -fn glob_runner(s: &str, command_builder: B, should_succeed: bool) +fn glob_runner(s: &str, args: &Args, command_builder: B, should_succeed: bool) where B: Fn(&Path) -> Option, { - let mut out = StandardStream::stdout(ColorChoice::Always); + let is_tty = std::io::stdout().is_terminal(); + let mut out = StandardStream::stdout(if args.force_color || is_tty { + ColorChoice::Always + } else { + ColorChoice::Never + }); let mut test_count = 0; let mut test_failures = 0; - let bless = std::env::args().any(|arg| arg == "--bless"); - let filter = std::env::args().nth(1); for entry in glob::glob(s).expect("Failed to read glob pattern") { test_count += 1; let entry = entry.unwrap(); - if let Some(ref filter) = filter { + if let Some(ref filter) = args.filter { if !entry.to_str().map(|entry| entry.contains(filter)).unwrap_or(false) { continue; } @@ -172,16 +201,24 @@ where let stderr = entry.with_extension("stderr"); let stdout = entry.with_extension("coma"); - write!(&mut out, "Testing {} ... ", entry.display()).unwrap(); + if !args.quiet { + write!(&mut out, "Testing {} ... ", entry.display()).unwrap(); + out.flush().unwrap(); + } - if bless { - out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap(); - writeln!(&mut out, "blessed").unwrap(); - out.reset().unwrap(); + if args.bless { + if !args.quiet { + out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap(); + writeln!(&mut out, "blessed").unwrap(); + out.reset().unwrap(); + } let (success, _) = - differ(output.clone(), &stdout, Some(&stderr), should_succeed).unwrap(); + differ(output.clone(), &stdout, Some(&stderr), should_succeed, is_tty).unwrap(); if !success { + if args.quiet { + write!(&mut out, "Testing {} ... ", entry.display()).unwrap(); + } out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap(); writeln!(&mut out, "failure").unwrap(); out.reset().unwrap(); @@ -199,13 +236,21 @@ where std::fs::write(stderr, &output.stderr).unwrap(); } } else { - let (success, mut buf) = - differ(output.clone(), &stdout, Some(&stderr), should_succeed).unwrap(); + let (success, buf) = + differ(output.clone(), &stdout, Some(&stderr), should_succeed, is_tty).unwrap(); if success { - out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap(); - writeln!(&mut out, "ok").unwrap(); + if is_tty { + // Move to beginning of line and clear line. + write!(out, "\x1b[G\x1b[2K").unwrap(); + } else if !args.quiet { + out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap(); + writeln!(out, "ok").unwrap(); + } } else { + if args.quiet { + write!(&mut out, "Testing {} ... ", entry.display()).unwrap(); + } out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap(); writeln!(&mut out, "failure").unwrap(); @@ -213,7 +258,6 @@ where }; out.reset().unwrap(); - buf.reset().unwrap(); let wrt = BufferWriter::stdout(ColorChoice::Always); wrt.print(&buf).unwrap(); } @@ -221,7 +265,8 @@ where if test_failures > 0 { out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap(); + writeln!(&mut out, "{test_failures} failures out of {test_count} tests").unwrap(); drop(out); - panic!("{} failures out of {} tests", test_failures, test_count); + std::process::exit(1); } }