Improve coverage output (#224)

* Reformat coverage and move export to command * Add simple html coverage report * Fix coverage badge * Add basic documentation
askimed · Jul 5, 2024 · b22274f · b22274f
1 parent 2bc23bc
commit b22274f
Show file tree

Hide file tree

Showing 8 changed files with 289 additions and 22 deletions.
diff --git a/docs/docs/cli/coverage.md b/docs/docs/cli/coverage.md
@@ -0,0 +1,19 @@
+# `coverage` command
+
+:octicons-tag-24: 0.9.0
+
+## Usage
+
+```
+nf-test coverage
+```
+
+The `coverage` command prints information about the number of Nextflow files that are covered by a test.
+
+### Optional Arguments
+
+#### `--csv <filename>`
+Writes a coverage report in csv format.
+
+#### `--html <filename>`
+Writes a coverage report in html format.
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -59,6 +59,7 @@ nav:
           - generate: docs/cli/generate.md
           - test: docs/cli/test.md
           - list: docs/cli/list.md
+          - coverage: docs/cli/coverage.md
           - clean: docs/cli/clean.md
       - Configuration: docs/configuration.md
       - Plugins:

diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java
@@ -1,12 +1,6 @@
 package com.askimed.nf.test;
 
-import com.askimed.nf.test.commands.CleanCommand;
-import com.askimed.nf.test.commands.GenerateTestsCommand;
-import com.askimed.nf.test.commands.InitCommand;
-import com.askimed.nf.test.commands.ListTestsCommand;
-import com.askimed.nf.test.commands.RunTestsCommand;
-import com.askimed.nf.test.commands.UpdatePluginsCommand;
-import com.askimed.nf.test.commands.VersionCommand;
+import com.askimed.nf.test.commands.*;
 
 import ch.qos.logback.classic.Level;
 import picocli.CommandLine;
@@ -35,6 +29,7 @@ public int run(String[] args) {
 		commandLine.addSubcommand("clean", new CleanCommand());
 		commandLine.addSubcommand("init", new InitCommand());
 		commandLine.addSubcommand("test", new RunTestsCommand());
+		commandLine.addSubcommand("coverage", new CoverageCommand());
 		commandLine.addSubcommand("list", new ListTestsCommand());
 		commandLine.addSubcommand("ls", new ListTestsCommand());
 		commandLine.addSubcommand("generate", new GenerateTestsCommand());

diff --git a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java
@@ -0,0 +1,96 @@
+package com.askimed.nf.test.commands;
+
+import com.askimed.nf.test.config.Config;
+import com.askimed.nf.test.lang.dependencies.Coverage;
+import com.askimed.nf.test.lang.dependencies.DependencyResolver;
+import com.askimed.nf.test.util.AnsiColors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Help.Visibility;
+import picocli.CommandLine.Option;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+@Command(name = "coverage")
+public class CoverageCommand extends AbstractCommand {
+
+	private static final String SHARD_STRATEGY_ROUND_ROBIN = "round-robin";
+
+	@Option(names = {
+			"--csv" }, description = "Write coverage results in csv format", required = false, showDefaultValue = Visibility.ALWAYS)
+	private String csv = null;
+
+	@Option(names = {
+			"--html" }, description = "Write coverage results in html format", required = false, showDefaultValue = Visibility.ALWAYS)
+	private String html = null;
+
+
+	@Option(names = { "--config",
+			"-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS)
+	private String configFilename = Config.FILENAME;
+
+	private static Logger log = LoggerFactory.getLogger(CoverageCommand.class);
+
+	@Override
+	public Integer execute() throws Exception {
+
+		List<File> scripts = new ArrayList<File>();
+		Config config = null;
+
+		try {
+
+			File defaultConfigFile = null;
+			boolean defaultWithTrace = true;
+			try {
+				File configFile = new File(configFilename);
+				if (configFile.exists()) {
+					log.info("Load config from file {}...", configFile.getAbsolutePath());
+					config = Config.parse(configFile);
+				} else {
+					System.out.println(AnsiColors.yellow("Warning: This pipeline has no nf-test config file."));
+					log.warn("No nf-test config file found.");
+				}
+
+			} catch (Exception e) {
+
+				System.out.println(AnsiColors.red("Error: Syntax errors in nf-test config file: " + e));
+				log.error("Parsing config file failed", e);
+				return 2;
+
+			}
+
+			File baseDir = new File(new File("").getAbsolutePath());
+			DependencyResolver resolver = new DependencyResolver(baseDir);
+			resolver.setFollowingDependencies(true);
+
+
+			if (config != null) {
+				resolver.buildGraph(config.getIgnore(), config.getTriggers());
+			} else {
+				resolver.buildGraph();
+			}
+
+			Coverage coverage = new Coverage(resolver).getAll();
+			if (csv != null) {
+				coverage.exportAsCsv(csv);
+			} else if (html != null) {
+				coverage.exportAsHtml(html);
+			} else {
+				coverage.printDetails();
+			}
+
+			return 0;
+
+		} catch (Throwable e) {
+
+			System.out.println(AnsiColors.red("Error: " + e));log.error("Running tests failed.", e);
+			return 1;
+
+		}
+
+	}
+
+}
diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java
@@ -188,7 +188,6 @@ public Integer execute() throws Exception {
 				return 2;
 			}
 
-			List<PathMatcher> ignorePatterns = new Vector<PathMatcher>();
 			File baseDir = new File(new File("").getAbsolutePath());
 			DependencyResolver resolver = new DependencyResolver(baseDir);
 			resolver.setFollowingDependencies(followDependencies);
@@ -231,20 +230,13 @@ public Integer execute() throws Exception {
 
 				AnsiText.printBulletList(scripts);
 
-				if (coverage) {
-					new Coverage(resolver).getByFiles(testPaths).print();
-				}
-
 			} else {
 				if (config != null) {
 					resolver.buildGraph(config.getIgnore(), config.getTriggers());
 				} else {
 					resolver.buildGraph();
 				}
 				scripts = resolver.findTestsByFiles(testPaths);
-				if (coverage) {
-					new Coverage(resolver).getAll().print();
-				}
 			}
 
 			if (graph != null) {
@@ -304,7 +296,15 @@ public Integer execute() throws Exception {
 				System.out.println(AnsiColors.yellow("Dry run mode activated: tests are not executed, just listed."));
 			}
 
-			return engine.execute();
+			int exitStatus = engine.execute();
+
+			if (coverage && findRelatedTests) {
+				new Coverage(resolver).getByFiles(testPaths).print();
+			} else if (coverage) {
+				new Coverage(resolver).getAll().print();
+			}
+
+			return exitStatus;
 
 		} catch (Throwable e) {
 

diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java
@@ -1,16 +1,31 @@
 package com.askimed.nf.test.lang.dependencies;
 
+import com.askimed.nf.test.commands.init.InitTemplates;
+import com.askimed.nf.test.core.TestExecutionResult;
+import com.askimed.nf.test.core.TestSuiteExecutionResult;
+import com.askimed.nf.test.core.reports.CsvReportWriter;
 import com.askimed.nf.test.util.AnsiColors;
+import com.askimed.nf.test.util.AnsiText;
+import com.askimed.nf.test.util.FileUtil;
+import com.opencsv.CSVWriter;
+import groovy.lang.Writable;
+import groovy.text.SimpleTemplateEngine;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Paths;
 import java.text.DecimalFormat;
-import java.util.List;
-import java.util.Vector;
+import java.text.DecimalFormatSymbols;
+import java.util.*;
 
 public class Coverage {
 
+    private static final String HTML_TEMPLATE = "coverage-report.html";
+
     private int coveredItems = 0;
 
     private DependencyGraph graph;
@@ -19,12 +34,15 @@ public class Coverage {
 
     private static Logger log = LoggerFactory.getLogger(Coverage.class);
 
+    private File baseDir = null;
+
     public Coverage(DependencyGraph graph) {
         this.graph = graph;
     }
 
     public Coverage(DependencyResolver resolver) {
         this.graph = resolver.getGraph();
+        baseDir = resolver.getBaseDir();
     }
 
     public void add(File file, boolean covered) {
@@ -57,6 +75,8 @@ public Coverage getAll(){
 
         }
 
+        items.sort(new CoverageItemSorter());
+
         long time1 = System.currentTimeMillis();
 
         log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0);
@@ -87,6 +107,8 @@ public Coverage getByFiles(List<File> files){
 
         }
 
+        items.sort(new CoverageItemSorter());
+
         long time1 = System.currentTimeMillis();
 
         log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0);
@@ -95,14 +117,89 @@ public Coverage getByFiles(List<File> files){
     }
 
     public void print() {
-        DecimalFormat decimalFormat = new DecimalFormat("#.##");
+        printLabel();
         System.out.println();
-        System.out.print("Coverage: " + getCoveredItems() + "/" + getItems().size());
-        System.out.println(" (" + decimalFormat.format(getCoveredItems() / (float) getItems().size() * 100) + "%)");
+    }
+
+    public void printDetails() {
+        System.out.println();
+        System.out.println("Files:");
         for (Coverage.CoverageItem item : getItems()) {
-            System.out.println("  - " + (item.isCovered() ? AnsiColors.green(item.getFile().getAbsolutePath()) : AnsiColors.red(item.getFile().getAbsolutePath())));
+            String label = getFileLabel(item.getFile());
+            System.out.println("  \u2022 " + (item.isCovered() ? AnsiColors.green(label) : AnsiColors.red(label)));
         }
         System.out.println();
+        printLabel();
+        System.out.println();
+    }
+
+    public String getFileLabel(File file) {
+        String label = file.getAbsolutePath();
+        if (baseDir != null) {
+            label = Paths.get(baseDir.getAbsolutePath()).relativize(file.toPath()).toString();
+        }
+        return label;
+    }
+
+    private void printLabel() {
+        float coverage = getCoveredItems() / (float) getItems().size();
+        System.out.print(getColor("COVERAGE:", coverage) + " " +  formatCoverage(coverage));
+        System.out.println( " [" + getCoveredItems() + " of " + getItems().size() + " files]");
+    }
+
+    public float getCoverage() {
+        return getCoveredItems() / (float) getItems().size();
+    }
+
+    private String getColor(String label, float value) {
+        if (value < 0.5) {
+            return AnsiColors.red(label);
+        } else if (value < 0.9) {
+            return AnsiColors.yellow(label);
+        } else {
+            return AnsiColors.green(label);
+        }
+    }
+
+    private String formatCoverage(float value) {
+        DecimalFormat decimalFormat = new DecimalFormat("#.##", DecimalFormatSymbols.getInstance(Locale.US));
+        return decimalFormat.format(value * 100) + "%";
+    }
+
+    public void exportAsCsv(String filename) throws IOException {
+        String[] header = new String[]{
+                "filename",
+                "covered",
+                "type"
+        };
+
+        CSVWriter writer = new CSVWriter(new FileWriter(new File(filename)));
+        writer.writeNext(header);
+        for (Coverage.CoverageItem item : getItems()) {
+            String[] line = new String[]{
+                item.getFile().getAbsolutePath(),
+                item.isCovered() + "",
+                "unknown"
+            };
+
+            writer.writeNext(line);
+        }
+
+        writer.close();
+        System.out.println();
+        printLabel();
+        System.out.println();
+        System.out.println("Wrote coverage report to file " + filename + "\n");
+
+    }
+
+    public void exportAsHtml(String filename) throws IOException, ClassNotFoundException {
+        Map<Object, Object> binding = new HashMap<Object, Object>();
+        binding.put("coverage", this);
+        URL templateUrl = Coverage.class.getResource(HTML_TEMPLATE);
+        SimpleTemplateEngine engine = new SimpleTemplateEngine();
+        Writable template = engine.createTemplate(templateUrl).make(binding);
+        FileUtil.write(new File(filename), template);
     }
 
     public static class CoverageItem {
@@ -111,6 +208,8 @@ public static class CoverageItem {
 
         private boolean covered = false;
 
+        //TODO: add number of tests??
+
         public CoverageItem(File file, boolean covered) {
             this.file = file;
             this.covered = covered;

diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java
@@ -0,0 +1,9 @@
+package com.askimed.nf.test.lang.dependencies;
+
+public class CoverageItemSorter implements java.util.Comparator<Coverage.CoverageItem> {
+
+    @Override
+    public int compare(Coverage.CoverageItem o1, Coverage.CoverageItem o2) {
+        return o1.getFile().getAbsolutePath().compareTo(o2.getFile().getAbsolutePath());
+    }
+}