Condense test logs by hiding successful tests using ANSI codes

- Disable ANSI codes when stdout is not a terminal
- Add special case for running in CI

creusot/Cargo.toml

@@ -16,7 +16,6 @@ petgraph = { version = "0.6", git = "https://github.com/xldenis/petgraph", rev =
 indexmap = { version = "1.7.0", features = ["serde"] }
 toml = "0.5.8"
 why3 = { path = "../why3", features = ["serialize"] }
-clap = { version = "4.2", features = ["derive", "env"] }
 creusot-metadata = { path = "../creusot-metadata" }
 include_dir = "0.7.3"
 tempdir = "0.3.7"
@@ -25,6 +24,7 @@ lazy_static = "1.4.0"
 pathdiff = "0.2"
 
 [dev-dependencies]
+clap = { version = "4.2", features = ["derive", "env"] }
 regex = "1.10.5"
 glob = "*"
 assert_cmd = "1.0"

creusot/tests/creusot-contracts.rs

@@ -1,11 +1,31 @@
-use std::{io::Write as _, path::PathBuf};
+use clap::Parser;
+use std::{
+    env,
+    io::{IsTerminal, Write as _},
+    path::PathBuf,
+};
 use termcolor::{BufferWriter, Color, ColorChoice, ColorSpec, StandardStream, WriteColor as _};
 
 mod diff;
 use diff::differ;
 
+#[derive(Debug, Parser)]
+struct Args {
+    /// Force color output
+    #[clap(long)]
+    force_color: bool,
+    /// Overwrite expected output files with actual output
+    #[clap(long)]
+    bless: bool,
+    /// Only run tests which contain this string
+    filter: Option<String>,
+}
+
 fn main() {
-    let bless = std::env::args().any(|arg| arg == "--bless");
+    let mut args = Args::parse();
+    if env::var("CI").is_ok() {
+        args.force_color = true;
+    }
     // Build creusot-rustc to make it available to cargo-creusot
     let _ = escargot::CargoBuild::new()
         .bin("creusot-rustc")
@@ -43,13 +63,21 @@ fn main() {
         "creusot-contracts",
     ]);
 
-    let output = cargo_creusot.output().unwrap();
-    let mut out = StandardStream::stdout(ColorChoice::Always);
+    let is_tty = std::io::stdout().is_terminal();
+    let mut out = StandardStream::stdout(if args.force_color || is_tty {
+        ColorChoice::Always
+    } else {
+        ColorChoice::Never
+    });
+
+    write!(out, "Testing creusot-contracts ... ").unwrap();
+    out.flush().unwrap();
 
+    let output = cargo_creusot.output().unwrap();
     let stdout = PathBuf::from("tests/creusot-contracts/creusot-contracts.coma");
 
     let mut failed = false;
-    if bless {
+    if args.bless {
         if output.stdout.is_empty() {
             panic!(
                 "creusot-contracts should have an output! stderr is:\n\n{}",
@@ -59,7 +87,7 @@ fn main() {
         out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap();
         writeln!(&mut out, "blessed").unwrap();
         out.reset().unwrap();
-        let (success, _) = differ(output.clone(), &stdout, None, true).unwrap();
+        let (success, _) = differ(output.clone(), &stdout, None, true, is_tty).unwrap();
 
         if !success {
             out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap();
@@ -69,7 +97,7 @@ fn main() {
 
         std::fs::write(stdout, &output.stdout).unwrap();
     } else {
-        let (success, mut buf) = differ(output.clone(), &stdout, None, true).unwrap();
+        let (success, buf) = differ(output.clone(), &stdout, None, true, is_tty).unwrap();
 
         if success {
             out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap();
@@ -82,7 +110,6 @@ fn main() {
         };
         out.reset().unwrap();
 
-        buf.reset().unwrap();
         let wrt = BufferWriter::stdout(ColorChoice::Always);
         wrt.print(&buf).unwrap();
     }

creusot/tests/diff.rs

@@ -19,8 +19,9 @@ pub fn differ(
     stdout: &Path,
     stderr: Option<&Path>,
     should_succeed: bool,
+    enable_color: bool,
 ) -> Result<(bool, Buffer), Box<dyn Error>> {
-    let mut buf = Buffer::ansi();
+    let mut buf = if enable_color { Buffer::ansi() } else { Buffer::no_color() };
     match output.clone().ok() {
         Ok(output) => {
             let expect_out = &std::fs::read(stdout).unwrap_or_else(|_| Vec::new());

creusot/tests/ui.rs

@@ -1,7 +1,8 @@
+use clap::Parser;
 use std::{
     env,
     fs::File,
-    io::{BufRead, BufReader, Write},
+    io::{BufRead, BufReader, IsTerminal, Write},
     path::{Path, PathBuf},
     process::Command,
 };
@@ -10,6 +11,21 @@ use termcolor::*;
 mod diff;
 use diff::{differ, normalize_file_path};
 
+#[derive(Debug, Parser)]
+struct Args {
+    /// Suppress all output other than failing test cases
+    #[clap(long)]
+    quiet: bool,
+    /// Force color output
+    #[clap(long)]
+    force_color: bool,
+    /// Overwrite expected output files with actual output
+    #[clap(long)]
+    bless: bool,
+    /// Only run tests which contain this string
+    filter: Option<String>,
+}
+
 fn main() {
     // Build `creusot-rustc` and `cargo-creusot`
 
@@ -55,12 +71,22 @@ fn main() {
         std::process::exit(1);
     }
 
-    should_fail("tests/should_fail/**/*.rs", |p| {
+    let mut args = Args::parse();
+    if env::var("CI").is_ok() {
+        args.quiet = true;
+        args.force_color = true;
+    }
+
+    should_fail("tests/should_fail/**/*.rs", &args, |p| {
         run_creusot(creusot_rustc.path(), p, &temp_file.to_string_lossy())
     });
-    should_succeed("tests/should_succeed/**/*.rs", |p| {
+    should_succeed("tests/should_succeed/**/*.rs", &args, |p| {
         run_creusot(creusot_rustc.path(), p, &temp_file.to_string_lossy())
     });
+
+    if !args.quiet {
+        println!("All tests passed!");
+    }
 }
 
 fn run_creusot(
@@ -130,36 +156,39 @@ fn run_creusot(
     Some(cmd)
 }
 
-fn should_succeed<B>(s: &str, b: B)
+fn should_succeed<B>(s: &str, args: &Args, b: B)
 where
     B: Fn(&Path) -> Option<std::process::Command>,
 {
-    glob_runner(s, b, true);
+    glob_runner(s, args, b, true);
 }
 
-fn should_fail<B>(s: &str, b: B)
+fn should_fail<B>(s: &str, args: &Args, b: B)
 where
     B: Fn(&Path) -> Option<std::process::Command>,
 {
-    glob_runner(s, b, false);
+    glob_runner(s, args, b, false);
 }
 
-fn glob_runner<B>(s: &str, command_builder: B, should_succeed: bool)
+fn glob_runner<B>(s: &str, args: &Args, command_builder: B, should_succeed: bool)
 where
     B: Fn(&Path) -> Option<std::process::Command>,
 {
-    let mut out = StandardStream::stdout(ColorChoice::Always);
+    let is_tty = std::io::stdout().is_terminal();
+    let mut out = StandardStream::stdout(if args.force_color || is_tty {
+        ColorChoice::Always
+    } else {
+        ColorChoice::Never
+    });
 
     let mut test_count = 0;
     let mut test_failures = 0;
-    let bless = std::env::args().any(|arg| arg == "--bless");
-    let filter = std::env::args().nth(1);
 
     for entry in glob::glob(s).expect("Failed to read glob pattern") {
         test_count += 1;
         let entry = entry.unwrap();
 
-        if let Some(ref filter) = filter {
+        if let Some(ref filter) = args.filter {
             if !entry.to_str().map(|entry| entry.contains(filter)).unwrap_or(false) {
                 continue;
             }
@@ -172,16 +201,24 @@ where
         let stderr = entry.with_extension("stderr");
         let stdout = entry.with_extension("coma");
 
-        write!(&mut out, "Testing {} ... ", entry.display()).unwrap();
+        if !args.quiet {
+            write!(&mut out, "Testing {} ... ", entry.display()).unwrap();
+            out.flush().unwrap();
+        }
 
-        if bless {
-            out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap();
-            writeln!(&mut out, "blessed").unwrap();
-            out.reset().unwrap();
+        if args.bless {
+            if !args.quiet {
+                out.set_color(ColorSpec::new().set_fg(Some(Color::Blue))).unwrap();
+                writeln!(&mut out, "blessed").unwrap();
+                out.reset().unwrap();
+            }
             let (success, _) =
-                differ(output.clone(), &stdout, Some(&stderr), should_succeed).unwrap();
+                differ(output.clone(), &stdout, Some(&stderr), should_succeed, is_tty).unwrap();
 
             if !success {
+                if args.quiet {
+                    write!(&mut out, "Testing {} ... ", entry.display()).unwrap();
+                }
                 out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap();
                 writeln!(&mut out, "failure").unwrap();
                 out.reset().unwrap();
@@ -199,29 +236,37 @@ where
                 std::fs::write(stderr, &output.stderr).unwrap();
             }
         } else {
-            let (success, mut buf) =
-                differ(output.clone(), &stdout, Some(&stderr), should_succeed).unwrap();
+            let (success, buf) =
+                differ(output.clone(), &stdout, Some(&stderr), should_succeed, is_tty).unwrap();
 
             if success {
-                out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap();
-                writeln!(&mut out, "ok").unwrap();
+                if is_tty {
+                    // Move to beginning of line and clear line.
+                    write!(out, "\x1b[G\x1b[2K").unwrap();
+                } else if !args.quiet {
+                    out.set_color(ColorSpec::new().set_fg(Some(Color::Green))).unwrap();
+                    writeln!(out, "ok").unwrap();
+                }
             } else {
+                if args.quiet {
+                    write!(&mut out, "Testing {} ... ", entry.display()).unwrap();
+                }
                 out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap();
                 writeln!(&mut out, "failure").unwrap();
 
                 test_failures += 1;
             };
             out.reset().unwrap();
 
-            buf.reset().unwrap();
             let wrt = BufferWriter::stdout(ColorChoice::Always);
             wrt.print(&buf).unwrap();
         }
     }
 
     if test_failures > 0 {
         out.set_color(ColorSpec::new().set_fg(Some(Color::Red))).unwrap();
+        writeln!(&mut out, "{test_failures} failures out of {test_count} tests").unwrap();
         drop(out);
-        panic!("{} failures out of {} tests", test_failures, test_count);
+        std::process::exit(1);
     }
 }