From 0ca66293dbae9dfae6bdd48f71c02a7bc3d69ce2 Mon Sep 17 00:00:00 2001 From: Jiri Date: Sun, 17 Nov 2024 19:59:32 +0100 Subject: [PATCH 01/25] xml results initial shot --- .../java/org/openjdk/jcstress/JCStress.java | 10 +- .../infra/grading/HTMLReportPrinter.java | 13 +- .../infra/grading/TextReportPrinter.java | 4 +- .../infra/grading/XMLReportPrinter.java | 408 ++++++++++++++++++ .../jcstress/infra/runners/TestConfig.java | 31 +- 5 files changed, 450 insertions(+), 16 deletions(-) create mode 100644 jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java index 37ac9e9b..c24e9084 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java @@ -30,6 +30,7 @@ import org.openjdk.jcstress.infra.grading.ExceptionReportPrinter; import org.openjdk.jcstress.infra.grading.TextReportPrinter; import org.openjdk.jcstress.infra.grading.HTMLReportPrinter; +import org.openjdk.jcstress.infra.grading.XMLReportPrinter; import org.openjdk.jcstress.infra.runners.TestConfig; import org.openjdk.jcstress.infra.runners.TestList; import org.openjdk.jcstress.os.*; @@ -157,11 +158,14 @@ public void parseResults() throws Exception { drc.dump(); drc.close(); - new TextReportPrinter(opts, collector).work(); - new HTMLReportPrinter(opts, collector, out).work(); + new TextReportPrinter(opts.verbosity(), collector).work(); + new HTMLReportPrinter(opts.getResultDest(), collector, out).work(); + new XMLReportPrinter(opts.getResultDest(), collector, out, false, XMLReportPrinter.isErrorAsFailure(), XMLReportPrinter.isTestsuiteUsed()).work(); + new XMLReportPrinter(opts.getResultDest(), collector, out, true, XMLReportPrinter.isErrorAsFailure(), XMLReportPrinter.isTestsuiteUsed()).work(); new ExceptionReportPrinter(collector).work(); } + private SortedSet computeActorCounts(Set tests) { SortedSet counts = new TreeSet<>(); for (String test : tests) { @@ -232,7 +236,7 @@ public int listTests(Options opts) { Set testsToPrint = new TreeSet<>(); for (TestConfig test : configsWithScheduler.configs) { if (opts.verbosity().printAllTests()) { - testsToPrint.add(test.toDetailedTest()); + testsToPrint.add(test.toDetailedTest(true)); } else { testsToPrint.add(test.name); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java index 2e06e954..39d8536f 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java @@ -25,7 +25,6 @@ package org.openjdk.jcstress.infra.grading; -import org.openjdk.jcstress.Options; import org.openjdk.jcstress.annotations.Expect; import org.openjdk.jcstress.infra.Status; import org.openjdk.jcstress.infra.TestInfo; @@ -57,19 +56,23 @@ public class HTMLReportPrinter { private final InProcessCollector collector; private int cellStyle = 1; - public HTMLReportPrinter(Options opts, InProcessCollector collector, PrintStream out) { + public HTMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out) { this.collector = collector; - this.resultDir = opts.getResultDest(); + this.resultDir = resultDir; File dir = new File(resultDir); dir.mkdirs(); - out.println(" HTML report generated at " + dir.getAbsolutePath() + File.separator + "index.html"); + out.println(" HTML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName()); + } + + public String getMainFileName() { + return "index.html"; } public void work() throws FileNotFoundException { List byName = ReportUtils.mergedByName(collector.getTestResults()); Collections.sort(byName, Comparator.comparing(TestResult::getName)); - PrintWriter output = new PrintWriter(resultDir + "/index.html"); + PrintWriter output = new PrintWriter(resultDir + File.separator + getMainFileName()); printHeader(output); diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java index ebb76220..815721ef 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java @@ -48,10 +48,10 @@ public class TextReportPrinter { private final PrintWriter pw; private final Set emittedTests; - public TextReportPrinter(Options opts, InProcessCollector collector) { + public TextReportPrinter(Verbosity verbosity, InProcessCollector collector) { this.collector = collector; this.pw = new PrintWriter(System.out, true); - this.verbosity = opts.verbosity(); + this.verbosity = verbosity; this.emittedTests = new HashSet<>(); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java new file mode 100644 index 00000000..ecfa8974 --- /dev/null +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jcstress.infra.grading; + + +import org.openjdk.jcstress.annotations.Expect; +import org.openjdk.jcstress.infra.Status; +import org.openjdk.jcstress.infra.TestInfo; +import org.openjdk.jcstress.infra.collectors.InProcessCollector; +import org.openjdk.jcstress.infra.collectors.TestResult; +import org.openjdk.jcstress.infra.runners.TestConfig; +import org.openjdk.jcstress.infra.runners.TestList; +import org.openjdk.jcstress.os.SchedulingClass; +import org.openjdk.jcstress.util.Multimap; +import org.openjdk.jcstress.util.StringUtils; +import org.openjdk.jcstress.vm.CompileMode; + +import java.awt.Color; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Predicate; + +/** + * Prints HTML reports. + * + * @author Aleksey Shipilev (aleksey.shipilev@oracle.com) + */ +public class XMLReportPrinter { + + public static final String ERROR_AS_FAILURE = "jcstress.xml.error2failure"; + public static final String USE_TESTSUITES = "jcstress.xml.testsuites"; + private final String resultDir; + private final InProcessCollector collector; + private final boolean sparse; + private final boolean errorAsFailure; + private final boolean useTestsuites; + + + public static boolean isErrorAsFailure() { + return "true".equals(System.getProperty(XMLReportPrinter.ERROR_AS_FAILURE)); + } + + public static boolean isTestsuiteUsed() { + return "true".equals(System.getProperty(XMLReportPrinter.USE_TESTSUITES)); + } + + public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out, boolean sparse, boolean errorAsFailure, boolean useTestsuites) { + //sparse true -ALL_MATCHING + //sparse false - as ALL_MATCHING_COMBINATIONS + //jednou smichat, jednou ne. Varovani kolik jich bude + //-xml true/false, defaults to sparse + this.collector = collector; + this.resultDir = resultDir; + this.sparse = sparse; + this.errorAsFailure = errorAsFailure; + this.useTestsuites = useTestsuites; + File dir = new File(resultDir); + dir.mkdirs(); + out.println(" " + getSparseString() + " XML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName() + ". " + ERROR_AS_FAILURE + "=" + errorAsFailure + ", " + USE_TESTSUITES + "=" + useTestsuites); + } + + public String getMainFileName() { + return "junit-" + getSparseString() + ".xml"; + } + + private String getSparseString() { + return sparse ? "sparse" : "full"; + } + + public void work() throws FileNotFoundException { + List byName = sparse ? ReportUtils.mergedByName(collector.getTestResults()) : new ArrayList<>(collector.getTestResults()); + Collections.sort(byName, Comparator.comparing(TestResult::getName)); + + PrintWriter output = new PrintWriter(resultDir + File.separator + getMainFileName()); + + { + int passedCount = 0; + int failedCount = 0; + int sanityFailedCount = 0; + for (TestResult result : byName) { + if (result.status() == Status.NORMAL) { + if (result.grading().isPassed) { + passedCount++; + } else { + failedCount++; + } + } else { + if (result.status() == Status.API_MISMATCH) { + sanityFailedCount++; + } else { + failedCount++; + } + } + } + + int totalCount = passedCount + failedCount + sanityFailedCount; + + output.println(""); + output.println(""); + } + output.println(" "); + output.println(" "); + output.println(" "); + output.println(" "); + } + + output.println(""); + + printXTests(byName, output, + "All tests", + "", + r -> true); + emitTestReports(ReportUtils.byName(collector.getTestResults()), output); + output.close(); + } + + private SortedMap getEnv(List ts) { + SortedMap env = new TreeMap<>(); + for (TestResult result : ts) { + if (result != null) { + for (Map.Entry kv : result.getEnv().entries().entrySet()) { + String key = kv.getKey(); + String value = kv.getValue(); + String lastV = env.get(key); + if (lastV == null) { + env.put(key, value); + } else { + // Some VMs have these keys pre-populated with the command line, + // which can have port definitions, PIDs, etc, and naturally + // clash from launch to launch. + if (key.equals("cmdLine")) continue; + if (key.equals("launcher")) continue; + + if (!lastV.equalsIgnoreCase(value)) { + System.err.println("Mismatched environment for key = " + key + ", was = " + lastV + ", now = " + value); + } + } + } + } + } + return env; + } + + + + private void printXTests(List byName, + PrintWriter output, + String header, + String subheader, + Predicate filterResults) { + output.println("*** " + header + " ***"); + output.println("" + subheader + ""); + boolean hadAnyTests = false; + for (TestResult result : byName) { + if (filterResults.test(result)) { + if (result.status() == Status.NORMAL) { + emitTest(output, result); + } else { + emitTestFailure(output, result); + } + hadAnyTests = true; + } + } + if (!hadAnyTests) { + output.println("None!"); + } + } + + public void emitTest(PrintWriter output, TestResult result) { + TestGrading grading = result.grading(); + if (grading.isPassed) { + output.println(" Passed - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + } else { + output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + } + + if (grading.hasInteresting) { + output.println(" was interesting"); + } + } + + public void emitTestFailure(PrintWriter output, TestResult result) { + output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + switch (result.status()) { + case API_MISMATCH: + output.println(" API MISMATCH - Sanity check failed, API mismatch?"); + break; + case TEST_ERROR: + case CHECK_TEST_ERROR: + output.println(" ERROR - Error while running the test"); + break; + case TIMEOUT_ERROR: + output.println(" ERROR - Timeout while running the test"); + break; + case VM_ERROR: + output.println(" VM ERROR - Error running the VM"); + break; + } + } + + public static String getRoughCount(TestResult r) { + long sum = r.getTotalCount(); + if (sum > 10) { + return "10^" + (int) Math.floor(Math.log10(sum)); + } else { + return String.valueOf(sum); + } + } + + private void emitTestReports(Multimap multiByName, PrintWriter local) { + multiByName.keys().parallelStream().forEach(name -> { + TestInfo test = TestList.getInfo(name); + local.println(resultDir + "/" + name + ".html would be..."); + emitTestReport(local, multiByName.get(name), test); + local.close(); + }); + } + + public void emitTestReport(PrintWriter o, Collection results, TestInfo test) { + o.println("subtests of: " + test.name()); + o.println(" Description and references:"); + o.println(" * " + test.description() + ""); + for (String ref : test.refs()) { + o.println(" " + ref); + } + + List sorted = new ArrayList<>(results); + sorted.sort(Comparator + .comparing((TestResult t) -> t.getConfig().getCompileMode()) + .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) + .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); + + + o.println(""); + for (Map.Entry entry : getEnv(sorted).entrySet()) { + o.println(""); + } + o.println(""); + + Set keys = new TreeSet<>(); + for (TestResult r : sorted) { + keys.addAll(r.getStateKeys()); + } + +// fixme use later? Defiitley ther emust be for (String key : keys) {for (TestResult r : sorted) {}} of results +// GradingResult c = r.grading().gradingResults.get(key); +// o.println("" + c.expect + ""); +// o.println("" + c.description + ""); + + + + for (TestResult r : sorted) { + String color = ReportUtils.statusToPassed(r) ? "green" : "red"; + String label = ReportUtils.statusToLabel(r); + o.println(color + " " + label + " " + r.getConfig().toDetailedTest(false)); //TODO, keep using the seed shading +// this is that multiplication. Probably just span it to failure if any? +// for (String key : keys) { +// GradingResult c = r.grading().gradingResults.get(key); +// if (c != null) { +// o.println("" + c.count + ""); +// } else { +// o.println("0"); +// } +// } + } + + + o.println("

Messages

"); + + for (TestResult r : sorted) { + if (!r.getMessages().isEmpty()) { + resultHeader(o, r); + o.println("
");
+                for (String data : r.getMessages()) {
+                    o.println(data);
+                }
+                o.println("
"); + o.println(); + } + } + + o.println("

VM Output Streams

"); + + for (TestResult r : sorted) { + if (!r.getVmOut().isEmpty()) { + resultHeader(o, r); + o.println("
");
+                for (String data : r.getVmOut()) {
+                    o.println(data);
+                }
+                o.println("
"); + o.println(); + } + } + + o.println("

VM Error Streams

"); + + for (TestResult r : sorted) { + if (!r.getVmErr().isEmpty()) { + resultHeader(o, r); + o.println("
");
+                for (String data : r.getVmErr()) {
+                    o.println(data);
+                }
+                o.println("
"); + o.println(); + } + } + + } + + private void resultHeader(PrintWriter o, TestResult r) { + TestConfig cfg = r.getConfig(); + o.println("

"); + o.println("

" + CompileMode.description(cfg.compileMode, cfg.actorNames) + "
"); + o.println("
" + SchedulingClass.description(cfg.shClass, cfg.actorNames) + "
"); + o.println(""); + if (!cfg.jvmArgs.isEmpty()) { + o.println("
" + cfg.jvmArgs + "
"); + } + o.println("

"); + } + + public String selectHTMLColor(Expect type, boolean isZero) { + String rgb = Integer.toHexString(selectColor(type, isZero).getRGB()); + return "#" + rgb.substring(2); + } + + public Color selectColor(Expect type, boolean isZero) { + switch (type) { + case ACCEPTABLE: + return isZero ? Color.LIGHT_GRAY : Color.GREEN; + case FORBIDDEN: + return isZero ? Color.LIGHT_GRAY : Color.RED; + case ACCEPTABLE_INTERESTING: + return isZero ? Color.LIGHT_GRAY : Color.CYAN; + case UNKNOWN: + return Color.RED; + default: + throw new IllegalStateException(); + } + } + +} diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java index 079feabb..4666e6c9 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java @@ -35,6 +35,7 @@ import java.io.PrintWriter; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; public class TestConfig implements Serializable { @@ -246,13 +247,12 @@ public void generateDirectives(PrintWriter pw, Verbosity verbosity) { } - public String toDetailedTest() { + public String getTestVariant(boolean seed) { //binaryName have correct $ instead of . in name; omitted //generatedRunnerName name with suffix (usually _Test_jcstress) omitted //super.toString() as TestConfig@hash - omitted - StringBuilder verboseOutput = new StringBuilder(name); - verboseOutput.append(" {") - .append(actorNames) + StringBuilder idString = new StringBuilder(); + idString.append(actorNames) .append(", spinLoopStyle: ").append(spinLoopStyle) .append(", threads: ").append(threads) .append(", forkId: ").append(forkId) @@ -262,8 +262,27 @@ public String toDetailedTest() { .append(", strideSize: ").append(strideSize) .append(", strideCount: ").append(strideCount) .append(", cpuMap: ").append(cpuMap) - .append(", ").append(jvmArgs) + .append(", ").append(seed ? jvmArgs : maskSeed(jvmArgs)); + return idString.toString(); + } + + private List maskSeed(List jvmArgs) { + List argsCopy = new ArrayList<>(jvmArgs.size()); + for (String arg : jvmArgs) { + if (arg.startsWith("-XX:StressSeed=")) { + argsCopy.add(arg.replaceAll("[0-9]+", "yyyyyyyy")); + } else { + argsCopy.add(arg); + } + } + return argsCopy; + } + + public String toDetailedTest(boolean seed) { + StringBuilder verboseOutput = new StringBuilder(name); + verboseOutput.append(" {") + .append(getTestVariant(seed)) .append("}"); return verboseOutput.toString(); } -} +} \ No newline at end of file From 1e9699ee69e77004c5f0681640dd1d10d65784e8 Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 18 Nov 2024 17:21:31 +0100 Subject: [PATCH 02/25] Removed cpuMap from detailed test output --- .../java/org/openjdk/jcstress/infra/runners/TestConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java index 4666e6c9..7826ba58 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java @@ -251,6 +251,7 @@ public String getTestVariant(boolean seed) { //binaryName have correct $ instead of . in name; omitted //generatedRunnerName name with suffix (usually _Test_jcstress) omitted //super.toString() as TestConfig@hash - omitted + //cpumap - null in listing, no reasonable toString method => omitted StringBuilder idString = new StringBuilder(); idString.append(actorNames) .append(", spinLoopStyle: ").append(spinLoopStyle) @@ -261,7 +262,6 @@ public String getTestVariant(boolean seed) { .append(", shClass: ").append(shClass) .append(", strideSize: ").append(strideSize) .append(", strideCount: ").append(strideCount) - .append(", cpuMap: ").append(cpuMap) .append(", ").append(seed ? jvmArgs : maskSeed(jvmArgs)); return idString.toString(); } From 49f1a4deb82af73522446cf865b4bbdef49aa5c3 Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 18 Nov 2024 17:33:00 +0100 Subject: [PATCH 03/25] hidden constructor used methods --- .../org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java | 2 +- .../org/openjdk/jcstress/infra/grading/XMLReportPrinter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java index 39d8536f..15338a25 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java @@ -64,7 +64,7 @@ public HTMLReportPrinter(String resultDir, InProcessCollector collector, PrintSt out.println(" HTML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName()); } - public String getMainFileName() { + private String getMainFileName() { return "index.html"; } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index ecfa8974..66f4f027 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -94,7 +94,7 @@ public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStr out.println(" " + getSparseString() + " XML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName() + ". " + ERROR_AS_FAILURE + "=" + errorAsFailure + ", " + USE_TESTSUITES + "=" + useTestsuites); } - public String getMainFileName() { + private String getMainFileName() { return "junit-" + getSparseString() + ".xml"; } From 2466f64083e0d00eb19f2bcaea3e0905234c4b8f Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 19 Nov 2024 18:21:49 +0100 Subject: [PATCH 04/25] Unhappy experiemnts to try traces from subtests the config is not in TEST_INFO the paralel stream chaneg isnecessary (before each wentr to its own file --- .../jcstress/infra/grading/ReportUtils.java | 7 +++++++ .../jcstress/infra/grading/XMLReportPrinter.java | 15 +++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java index b4bc4121..40845779 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java @@ -84,6 +84,13 @@ public static Multimap byName(Collection src) { } return result; } + public static Multimap byDetailedName(Collection src) { + Multimap result = new HashMultimap<>(); + for (TestResult r : mergedByConfig(src)) { + result.put(r.getConfig().toDetailedTest(false), r); + } + return result; + } private static TestResult merged(TestConfig config, Collection mergeable) { Counter counter = new Counter<>(); diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 66f4f027..894bbfe6 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -174,7 +174,11 @@ public void work() throws FileNotFoundException { "All tests", "", r -> true); - emitTestReports(ReportUtils.byName(collector.getTestResults()), output); + if (sparse) { + emitTestReports(ReportUtils.byName(collector.getTestResults()), output); + } else { + emitTestReports(ReportUtils.byDetailedName(collector.getTestResults()), output); + } output.close(); } @@ -272,8 +276,8 @@ public static String getRoughCount(TestResult r) { } private void emitTestReports(Multimap multiByName, PrintWriter local) { - multiByName.keys().parallelStream().forEach(name -> { - TestInfo test = TestList.getInfo(name); + multiByName.keys().stream().forEach(name -> { + TestInfo test = TestList.getInfo(name.split(" ")[0]); local.println(resultDir + "/" + name + ".html would be..."); emitTestReport(local, multiByName.get(name), test); local.close(); @@ -385,11 +389,6 @@ private void resultHeader(PrintWriter o, TestResult r) { o.println("

"); } - public String selectHTMLColor(Expect type, boolean isZero) { - String rgb = Integer.toHexString(selectColor(type, isZero).getRGB()); - return "#" + rgb.substring(2); - } - public Color selectColor(Expect type, boolean isZero) { switch (type) { case ACCEPTABLE: From ee5c33a9c2725541906074177f26822c70f719f3 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 22 Nov 2024 14:57:36 +0100 Subject: [PATCH 05/25] use name of proeprty --- .../org/openjdk/jcstress/infra/grading/XMLReportPrinter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 894bbfe6..6eecd83e 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -148,8 +148,8 @@ public void work() throws FileNotFoundException { output.println(" "); } output.println(" "); - output.println(" "); - output.println(" "); + output.println(" "); + output.println(" "); output.println(" "); } From 17e8cc5b7686b1fab68c272e418ce4e7a1f37111 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 22 Nov 2024 20:37:47 +0100 Subject: [PATCH 06/25] Changed approach to iterate ower merged results. It looks good now, but it may that changes on top of prevous commit was missbeahving due to wrongly updated classpath:-/ t --- .../infra/grading/HTMLReportPrinter.java | 14 +- .../infra/grading/XMLReportPrinter.java | 275 ++++++++++-------- .../jcstress/infra/runners/TestConfig.java | 8 +- 3 files changed, 159 insertions(+), 138 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java index 15338a25..5000efe2 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java @@ -174,7 +174,7 @@ public void work() throws FileNotFoundException { emitTestReports(ReportUtils.byName(collector.getTestResults())); } - private SortedMap getEnv(List ts) { + static SortedMap getEnv(List ts) { SortedMap env = new TreeMap<>(); for (TestResult result : ts) { if (result != null) { @@ -356,10 +356,7 @@ public void emitTestReport(PrintWriter o, Collection results, TestIn } List sorted = new ArrayList<>(results); - sorted.sort(Comparator - .comparing((TestResult t) -> t.getConfig().getCompileMode()) - .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) - .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); + resultsOrder(sorted); o.println("

Environment

"); o.println(""); @@ -495,6 +492,13 @@ public void emitTestReport(PrintWriter o, Collection results, TestIn printFooter(o); } + static void resultsOrder(List sorted) { + sorted.sort(Comparator + .comparing((TestResult t) -> t.getConfig().getCompileMode()) + .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) + .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); + } + private void resultHeader(PrintWriter o, TestResult r) { TestConfig cfg = r.getConfig(); o.println("

"); diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 6eecd83e..3853cb4b 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -42,6 +42,7 @@ import java.io.FileNotFoundException; import java.io.PrintStream; import java.io.PrintWriter; +import java.net.InetAddress; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -130,6 +131,12 @@ public void work() throws FileNotFoundException { int totalCount = passedCount + failedCount + sanityFailedCount; + String hostname="localhost"; + try { + hostname = InetAddress.getLocalHost().getHostName(); + }catch (Exception ex) { + //no interest + } output.println(""); output.println(""); output.println(" "); } +// we have create dsummary, lets try to prnt the rest from merged info + byName = ReportUtils.mergedByName(collector.getTestResults()); + Collections.sort(byName, Comparator.comparing(TestResult::getName)); output.println(""); - printXTests(byName, output, - "All tests", - "", - r -> true); - if (sparse) { - emitTestReports(ReportUtils.byName(collector.getTestResults()), output); - } else { - emitTestReports(ReportUtils.byDetailedName(collector.getTestResults()), output); - } + emitTestReports(ReportUtils.byName(collector.getTestResults()), output); output.close(); } - private SortedMap getEnv(List ts) { - SortedMap env = new TreeMap<>(); - for (TestResult result : ts) { - if (result != null) { - for (Map.Entry kv : result.getEnv().entries().entrySet()) { - String key = kv.getKey(); - String value = kv.getValue(); - String lastV = env.get(key); - if (lastV == null) { - env.put(key, value); - } else { - // Some VMs have these keys pre-populated with the command line, - // which can have port definitions, PIDs, etc, and naturally - // clash from launch to launch. - if (key.equals("cmdLine")) continue; - if (key.equals("launcher")) continue; - - if (!lastV.equalsIgnoreCase(value)) { - System.err.println("Mismatched environment for key = " + key + ", was = " + lastV + ", now = " + value); - } - } - } - } - } - return env; - } - - - private void printXTests(List byName, PrintWriter output, String header, @@ -277,7 +251,7 @@ public static String getRoughCount(TestResult r) { private void emitTestReports(Multimap multiByName, PrintWriter local) { multiByName.keys().stream().forEach(name -> { - TestInfo test = TestList.getInfo(name.split(" ")[0]); + TestInfo test = TestList.getInfo(name); local.println(resultDir + "/" + name + ".html would be..."); emitTestReport(local, multiByName.get(name), test); local.close(); @@ -285,93 +259,138 @@ private void emitTestReports(Multimap multiByName, PrintWrit } public void emitTestReport(PrintWriter o, Collection results, TestInfo test) { - o.println("subtests of: " + test.name()); - o.println(" Description and references:"); - o.println(" * " + test.description() + ""); - for (String ref : test.refs()) { - o.println(" " + ref); - } - - List sorted = new ArrayList<>(results); - sorted.sort(Comparator - .comparing((TestResult t) -> t.getConfig().getCompileMode()) - .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) - .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); - - - o.println(""); - for (Map.Entry entry : getEnv(sorted).entrySet()) { - o.println(""); - } - o.println(""); - - Set keys = new TreeSet<>(); - for (TestResult r : sorted) { - keys.addAll(r.getStateKeys()); - } - -// fixme use later? Defiitley ther emust be for (String key : keys) {for (TestResult r : sorted) {}} of results -// GradingResult c = r.grading().gradingResults.get(key); -// o.println("

"); -// o.println(""); - - - - for (TestResult r : sorted) { - String color = ReportUtils.statusToPassed(r) ? "green" : "red"; - String label = ReportUtils.statusToLabel(r); - o.println(color + " " + label + " " + r.getConfig().toDetailedTest(false)); //TODO, keep using the seed shading -// this is that multiplication. Probably just span it to failure if any? -// for (String key : keys) { -// GradingResult c = r.grading().gradingResults.get(key); -// if (c != null) { -// o.println(""); -// } else { -// o.println(""); -// } -// } - } - + //in sparse mode we print only test.name as test, with result based on cumulative + //otherwise we weill be printing only its individual combinations (to mach the summary) + if (sparse) { + List sorted = new ArrayList<>(results); + HTMLReportPrinter.resultsOrder(sorted); + + o.println(" "); + for (String ref : test.refs()) { + o.println(" "); + } + for (Map.Entry entry : HTMLReportPrinter.getEnv(sorted).entrySet()) { + o.println(" "); + } + o.println(" "); - o.println("

Messages

"); - for (TestResult r : sorted) { - if (!r.getMessages().isEmpty()) { - resultHeader(o, r); - o.println("
");
-                for (String data : r.getMessages()) {
-                    o.println(data);
+            Set keys = new TreeSet<>();
+            for (TestResult r : sorted) {
+                keys.addAll(r.getStateKeys());
+            }
+            for (TestResult r : sorted) {
+                o.println("");
+                o.println(r.getConfig().toDetailedTest(false));
+                String color = ReportUtils.statusToPassed(r) ? "green" : "red";
+                String label = ReportUtils.statusToLabel(r);
+                o.println(color + " - " + label);
+
+                for (String key : keys) {
+                    GradingResult c = r.grading().gradingResults.get(key);
+                    if (c != null) {
+                        o.println(selectColor(c.expect, c.count == 0) + "/" + c.count + "");
+                    } else {
+                        o.println(selectColor(Expect.ACCEPTABLE, true) + "/0");
+                    }
                 }
-                o.println("
"); - o.println(); + o.println(""); } - } - - o.println("

VM Output Streams

"); - for (TestResult r : sorted) { - if (!r.getVmOut().isEmpty()) { - resultHeader(o, r); - o.println("
");
-                for (String data : r.getVmOut()) {
-                    o.println(data);
+            o.println("");
+            for (TestResult r : sorted) {
+                if (!r.getMessages().isEmpty()) {
+                    resultHeader(o, r);
+                    for (String data : r.getMessages()) {
+                        o.println(data);
+                    }
+                    o.println();
+                }
+                if (!r.getVmOut().isEmpty()) {
+                    resultHeader(o, r);
+                    for (String data : r.getVmOut()) {
+                        o.println(data);
+                    }
+                    o.println();
                 }
-                o.println("
"); - o.println(); } - } - - o.println("

VM Error Streams

"); + o.println(""); + o.println(""); + for (TestResult r : sorted) { + if (!r.getVmErr().isEmpty()) { + resultHeader(o, r); TestConfig cfg = r.getConfig(); + for (String data : r.getVmErr()) { + o.println(data); + } + o.println(); + } + } + o.println("\n"); - for (TestResult r : sorted) { - if (!r.getVmErr().isEmpty()) { - resultHeader(o, r); - o.println("
");
-                for (String data : r.getVmErr()) {
-                    o.println(data);
+            o.println("");
+        } else {
+            List sorted = new ArrayList<>(results);
+            HTMLReportPrinter.resultsOrder(sorted);
+            for (TestResult r : sorted) {
+                o.println("  ");
+                for (String ref : test.refs()) {
+                    o.println("          ");
                 }
-                o.println("
"); - o.println(); + for (Map.Entry entry : HTMLReportPrinter.getEnv(sorted).entrySet()) { + o.println(" "); + } + o.println(" "); + + Set keys = new TreeSet<>(); + keys.addAll(r.getStateKeys()); + o.println(""); + + TestConfig cfg = r.getConfig(); + o.println(r.getConfig().toDetailedTest(false)); + String color = ReportUtils.statusToPassed(r) ? "green" : "red"; + String label = ReportUtils.statusToLabel(r); + o.println(color + " - " + label); + + for (String key : keys) { + GradingResult c = r.grading().gradingResults.get(key); + if (c != null) { + o.println(selectColor(c.expect, c.count == 0) + "/" + c.count + ""); + } else { + o.println(selectColor(Expect.ACCEPTABLE, true) + "/0"); + } + } + o.println(""); + o.println(""); + if (!r.getMessages().isEmpty()) { + resultHeader(o, r); + for (String data : r.getMessages()) { + o.println(data); + } + o.println(); + } + if (!r.getVmOut().isEmpty()) { + resultHeader(o, r); + for (String data : r.getVmOut()) { + o.println(data); + } + o.println(); + } + o.println(""); + o.println(""); + if (!r.getVmErr().isEmpty()) { + resultHeader(o, r); + for (String data : r.getVmErr()) { + o.println(data); + } + o.println(); + } + o.println("\n"); + + o.println(""); } } @@ -379,26 +398,24 @@ public void emitTestReport(PrintWriter o, Collection results, TestIn private void resultHeader(PrintWriter o, TestResult r) { TestConfig cfg = r.getConfig(); - o.println("

"); - o.println("

" + CompileMode.description(cfg.compileMode, cfg.actorNames) + "
"); - o.println("
" + SchedulingClass.description(cfg.shClass, cfg.actorNames) + "
"); + o.println("CompileMode: " + CompileMode.description(cfg.compileMode, cfg.actorNames)); + o.println("SchedulingClass" + SchedulingClass.description(cfg.shClass, cfg.actorNames)); o.println(""); if (!cfg.jvmArgs.isEmpty()) { - o.println("
" + cfg.jvmArgs + "
"); + o.println("jvmargs:" + cfg.jvmArgs); } - o.println("

"); } - public Color selectColor(Expect type, boolean isZero) { + public String selectColor(Expect type, boolean isZero) { switch (type) { case ACCEPTABLE: - return isZero ? Color.LIGHT_GRAY : Color.GREEN; + return isZero ? "LIGHT_GRAY" : "GREEN"; case FORBIDDEN: - return isZero ? Color.LIGHT_GRAY : Color.RED; + return isZero ? "LIGHT_GRAY" : "RED"; case ACCEPTABLE_INTERESTING: - return isZero ? Color.LIGHT_GRAY : Color.CYAN; + return isZero ? "LIGHT_GRAY" : "CYAN"; case UNKNOWN: - return Color.RED; + return "RED"; default: throw new IllegalStateException(); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java index 7826ba58..b771a1bf 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java @@ -247,7 +247,7 @@ public void generateDirectives(PrintWriter pw, Verbosity verbosity) { } - public String getTestVariant(boolean seed) { + public String getTestVariant(boolean keepSeed) { //binaryName have correct $ instead of . in name; omitted //generatedRunnerName name with suffix (usually _Test_jcstress) omitted //super.toString() as TestConfig@hash - omitted @@ -262,7 +262,7 @@ public String getTestVariant(boolean seed) { .append(", shClass: ").append(shClass) .append(", strideSize: ").append(strideSize) .append(", strideCount: ").append(strideCount) - .append(", ").append(seed ? jvmArgs : maskSeed(jvmArgs)); + .append(", ").append(keepSeed ? jvmArgs : maskSeed(jvmArgs)); return idString.toString(); } @@ -278,10 +278,10 @@ private List maskSeed(List jvmArgs) { return argsCopy; } - public String toDetailedTest(boolean seed) { + public String toDetailedTest(boolean keepSeed) { StringBuilder verboseOutput = new StringBuilder(name); verboseOutput.append(" {") - .append(getTestVariant(seed)) + .append(getTestVariant(keepSeed)) .append("}"); return verboseOutput.toString(); } From 73e04b67a827f1abcb1bb693334ea3c5398e58be Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 25 Nov 2024 20:59:02 +0100 Subject: [PATCH 07/25] Reorganized and cleaned up as it finally do what it should a lot of todo - testsuites and pertestsuite stats - messages to failures - decide what to do with comments - with message sin failures probably nothing, and no comments needs to be there - drop its property - validate --- .../java/org/openjdk/jcstress/JCStress.java | 4 +- .../infra/grading/XMLReportPrinter.java | 231 ++++++++++++------ .../jcstress/infra/runners/TestConfig.java | 11 +- 3 files changed, 173 insertions(+), 73 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java index c24e9084..d89c5dec 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java @@ -160,8 +160,8 @@ public void parseResults() throws Exception { new TextReportPrinter(opts.verbosity(), collector).work(); new HTMLReportPrinter(opts.getResultDest(), collector, out).work(); - new XMLReportPrinter(opts.getResultDest(), collector, out, false, XMLReportPrinter.isErrorAsFailure(), XMLReportPrinter.isTestsuiteUsed()).work(); - new XMLReportPrinter(opts.getResultDest(), collector, out, true, XMLReportPrinter.isErrorAsFailure(), XMLReportPrinter.isTestsuiteUsed()).work(); + new XMLReportPrinter(opts.getResultDest(), collector, out, false).work(); + new XMLReportPrinter(opts.getResultDest(), collector, out, true).work(); new ExceptionReportPrinter(collector).work(); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 3853cb4b..8552a2ba 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -36,13 +36,27 @@ import org.openjdk.jcstress.util.Multimap; import org.openjdk.jcstress.util.StringUtils; import org.openjdk.jcstress.vm.CompileMode; - -import java.awt.Color; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.Validator; import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -52,7 +66,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.Predicate; @@ -63,24 +76,17 @@ */ public class XMLReportPrinter { - public static final String ERROR_AS_FAILURE = "jcstress.xml.error2failure"; - public static final String USE_TESTSUITES = "jcstress.xml.testsuites"; + public static final String ERROR_AS = "jcstress.report.xml.errorAs"; //pass/fail + public static final String USE_TESTSUITES = "jcstress.report.xml.sparse.testsuites"; + public static final String DUPLICATE_PROPERTIES = "jcstress.report.xml.properties.dupliate"; + public static final String VALIDATE = "jcstress.report.xml.validate"; + public static final String NO_COMMENTS = "jcstress.report.xml.nocomments"; private final String resultDir; private final InProcessCollector collector; private final boolean sparse; - private final boolean errorAsFailure; - private final boolean useTestsuites; - + private final PrintStream out; - public static boolean isErrorAsFailure() { - return "true".equals(System.getProperty(XMLReportPrinter.ERROR_AS_FAILURE)); - } - - public static boolean isTestsuiteUsed() { - return "true".equals(System.getProperty(XMLReportPrinter.USE_TESTSUITES)); - } - - public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out, boolean sparse, boolean errorAsFailure, boolean useTestsuites) { + public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out, boolean sparse) { //sparse true -ALL_MATCHING //sparse false - as ALL_MATCHING_COMBINATIONS //jednou smichat, jednou ne. Varovani kolik jich bude @@ -88,11 +94,68 @@ public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStr this.collector = collector; this.resultDir = resultDir; this.sparse = sparse; - this.errorAsFailure = errorAsFailure; - this.useTestsuites = useTestsuites; File dir = new File(resultDir); dir.mkdirs(); - out.println(" " + getSparseString() + " XML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName() + ". " + ERROR_AS_FAILURE + "=" + errorAsFailure + ", " + USE_TESTSUITES + "=" + useTestsuites); + out.println(" " + getSparseString() + " XML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName()); + this.out = out; + } + + public static ErrorAs getErrorAs() { + if (System.getProperty(XMLReportPrinter.ERROR_AS) == null) { + return ErrorAs.error; + } + return Enum.valueOf(ErrorAs.class, System.getProperty(XMLReportPrinter.ERROR_AS)); + } + + public static boolean isTestsuiteUsed() { + return System.getProperty(XMLReportPrinter.USE_TESTSUITES) != null; + } + + public static boolean isValidate() { + return System.getProperty(XMLReportPrinter.VALIDATE) != null; + } + + public static boolean isDuplicateProperties() { + return System.getProperty(XMLReportPrinter.DUPLICATE_PROPERTIES) != null; + } + + public static boolean isNoComments() { + return System.getProperty(XMLReportPrinter.NO_COMMENTS) != null; + } + + private static void printBaseProperties(List sorted, PrintWriter o) { + for (Map.Entry entry : HTMLReportPrinter.getEnv(sorted).entrySet()) { + o.println(" "); + } + } + + public static String getRoughCount(TestResult r) { + long sum = r.getTotalCount(); + if (sum > 10) { + return "10^" + (int) Math.floor(Math.log10(sum)); + } else { + return String.valueOf(sum); + } + } + + private static void printSeed(PrintWriter o, TestResult r) { + if (r.getConfig().getSeed() != null) { + o.println(" "); + } + } + + private static void printRefs(PrintWriter o, TestInfo test) { + for (String ref : test.refs()) { + if (ref != null) { + o.println(" "); + } + } + } + + private static void printDescription(PrintWriter o, TestInfo test) { + if (test.description() != null) { + o.println(" "); + } } private String getMainFileName() { @@ -107,7 +170,8 @@ public void work() throws FileNotFoundException { List byName = sparse ? ReportUtils.mergedByName(collector.getTestResults()) : new ArrayList<>(collector.getTestResults()); Collections.sort(byName, Comparator.comparing(TestResult::getName)); - PrintWriter output = new PrintWriter(resultDir + File.separator + getMainFileName()); + String filePath = resultDir + File.separator + getMainFileName(); + PrintWriter output = new PrintWriter(filePath); { int passedCount = 0; @@ -131,33 +195,35 @@ public void work() throws FileNotFoundException { int totalCount = passedCount + failedCount + sanityFailedCount; - String hostname="localhost"; + String hostname = "localhost"; try { hostname = InetAddress.getLocalHost().getHostName(); - }catch (Exception ex) { + } catch (Exception ex) { //no interest } output.println(""); + //in case of testsuites used + //consuilt + //check whether both writings ar eok for jtreg plugin output.println(""); + " timestamp='" + new Date().toString() + "' " + + " hostname='" + hostname + "'>"); } { - SortedMap env = HTMLReportPrinter.getEnv(byName); output.println(" "); - for (Map.Entry entry : env.entrySet()) { - output.println(" "); - } - output.println(" "); - output.println(" "); - output.println(" "); + printBaseProperties(byName, output); + output.println(" "); + output.println(" "); + output.println(" "); + output.println(" "); + output.println(" "); output.println(" "); } // we have create dsummary, lets try to prnt the rest from merged info @@ -182,7 +248,12 @@ public void work() throws FileNotFoundException { output.println("-->"); emitTestReports(ReportUtils.byName(collector.getTestResults()), output); + output.println(""); + output.flush(); output.close(); + if (isValidate()) { + validate(filePath); + } } private void printXTests(List byName, @@ -211,9 +282,9 @@ private void printXTests(List byName, public void emitTest(PrintWriter output, TestResult result) { TestGrading grading = result.grading(); if (grading.isPassed) { - output.println(" Passed - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + output.println(" Passed - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); } else { - output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); } if (grading.hasInteresting) { @@ -222,7 +293,7 @@ public void emitTest(PrintWriter output, TestResult result) { } public void emitTestFailure(PrintWriter output, TestResult result) { - output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " +getRoughCount(result)); + output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); switch (result.status()) { case API_MISMATCH: output.println(" API MISMATCH - Sanity check failed, API mismatch?"); @@ -240,54 +311,38 @@ public void emitTestFailure(PrintWriter output, TestResult result) { } } - public static String getRoughCount(TestResult r) { - long sum = r.getTotalCount(); - if (sum > 10) { - return "10^" + (int) Math.floor(Math.log10(sum)); - } else { - return String.valueOf(sum); - } - } - private void emitTestReports(Multimap multiByName, PrintWriter local) { multiByName.keys().stream().forEach(name -> { TestInfo test = TestList.getInfo(name); local.println(resultDir + "/" + name + ".html would be..."); emitTestReport(local, multiByName.get(name), test); - local.close(); }); } public void emitTestReport(PrintWriter o, Collection results, TestInfo test) { //in sparse mode we print only test.name as test, with result based on cumulative - //otherwise we weill be printing only its individual combinations (to mach the summary) + //otherwise we will be printing only its individual combinations (to mach the summary) if (sparse) { List sorted = new ArrayList<>(results); HTMLReportPrinter.resultsOrder(sorted); - - o.println(" "); o.println(" "); - o.println(" "); - for (String ref : test.refs()) { - o.println(" "); - } - for (Map.Entry entry : HTMLReportPrinter.getEnv(sorted).entrySet()) { - o.println(" "); + printDescription(o, test); + printRefs(o, test); + if (isDuplicateProperties()) { + printBaseProperties(sorted, o); } o.println(" "); - - Set keys = new TreeSet<>(); for (TestResult r : sorted) { keys.addAll(r.getStateKeys()); } + o.println(""); //or error //or "); - emitTestReports(ReportUtils.byName(collector.getTestResults()), output); output.println(""); output.flush(); @@ -298,8 +274,8 @@ private static String printTestSuiteHeader(List results, String name " failures='" + reordered.get(JunitResult.failure).size() + "'" + " errors='" + reordered.get(JunitResult.error).size() + "'" + " skipped='" + reordered.get(JunitResult.skipped).size() + "' " + - " passed='" + reordered.get(JunitResult.pass).size() + "'" + - " time='" + 0/*fixme getRoughCount?*/ + "'" + + //" passed='" + reordered.get(JunitResult.pass).size() + "'" + would nto pass validation + " time='" + getTime(results, false) + "'" + " timestamp='" + new Date().toString() + "' " + " hostname='" + hostname + "'>"; } @@ -364,7 +340,7 @@ private void emitTestReport(PrintWriter outw, Collection results, Te if (sparse) { List sorted = new ArrayList<>(results); HTMLReportPrinter.resultsOrder(sorted); - outw.println(" "); + outw.println(" "); printPropertiesPerTest(outw, test, null, sorted); printMainTestBody(outw, sorted, true); outw.println(""); @@ -380,7 +356,7 @@ private void emitTestReport(PrintWriter outw, Collection results, Te if (isTestsuiteUsed() && isStripNames()) { testName = r.getConfig().getTestVariant(false); } - outw.println(" "); + outw.println(" "); printPropertiesPerTest(outw, test, r, sorted); printMainTestBody(outw, Arrays.asList(r), null); outw.println(""); @@ -392,6 +368,22 @@ private void emitTestReport(PrintWriter outw, Collection results, Te } + private static String getTime(Collection results, boolean toNicestring) { + double sum = 0; + for (TestResult resul : results) { + sum += resul.getTotalCount(); + } + if (toNicestring) { + if (sum > 10) { + return "10^" + (long) Math.floor((Math.log10(sum))); + } else { + return String.valueOf(sum); + } + } else { + return ""+sum; + } + } + private static void printPropertiesPerTest(PrintWriter outw, TestInfo test, TestResult result, List sorted) { String props = printPropertiesPerTest(test, result, sorted); if (!props.isEmpty()) { @@ -417,6 +409,17 @@ private static String printPropertiesPerTest(TestInfo test, TestResult result, L sb.append(seed); } } + if (result != null) { + if (result.grading().hasInteresting) { + sb.append(" " + printProperty("interesting", true) + "\n"); + } + } else { + boolean anyIntresting = getAnyInteresting(sorted); + if (anyIntresting) { + sb.append(" " + printProperty("interesting", true) + "\n"); + } + } + if (isDuplicateProperties()) { String baseProps = getBaseProperties(sorted); if (baseProps != null && !baseProps.isEmpty()) { @@ -426,6 +429,15 @@ private static String printPropertiesPerTest(TestInfo test, TestResult result, L return sb.toString(); } + private static boolean getAnyInteresting(List results) { + for (TestResult result : results) { + if (result.grading().hasInteresting) { + return true; + } + } + return false; + } + private static void printMainTestBody(PrintWriter outw, List results, Boolean header) { Set keys = new TreeSet<>(); for (TestResult result : results) { @@ -620,72 +632,4 @@ public static JunitResult testToJunitResult(TestResult result) { } } - /// /// /////////candidates to remove - /// /// /////////candidates to remove - /// /// /////////candidates to remove - - private static String getRoughCount(TestResult r) { - long sum = r.getTotalCount(); - if (sum > 10) { - return "10^" + (int) Math.floor(Math.log10(sum)); - } else { - return String.valueOf(sum); - } - } - - private void printXTests(List byName, - PrintWriter output, - String header, - String subheader, - Predicate filterResults) { - output.println("*** " + header + " ***"); - output.println("" + subheader + ""); - boolean hadAnyTests = false; - for (TestResult result : byName) { - if (filterResults.test(result)) { - if (result.status() == Status.NORMAL) { - emitTest(output, result); - } else { - emitTestFailure(output, result); - } - hadAnyTests = true; - } - } - if (!hadAnyTests) { - output.println("None!"); - } - } - - private void emitTest(PrintWriter output, TestResult result) { - TestGrading grading = result.grading(); - if (grading.isPassed) { - output.println(" Passed - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); - } else { - output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); - } - - if (grading.hasInteresting) { - output.println(" was interesting"); - } - } - - private void emitTestFailure(PrintWriter output, TestResult result) { - output.println(" FAILED - " + StringUtils.chunkName(result.getName()) + " " + getRoughCount(result)); - switch (result.status()) { - case API_MISMATCH: - output.println(" API MISMATCH - Sanity check failed, API mismatch?"); - break; - case TEST_ERROR: - case CHECK_TEST_ERROR: - output.println(" ERROR - Error while running the test"); - break; - case TIMEOUT_ERROR: - output.println(" ERROR - Timeout while running the test"); - break; - case VM_ERROR: - output.println(" VM ERROR - Error running the VM"); - break; - } - } - -} \ No newline at end of file +} From 077393b703927e43eceadd22c3b6865fb7f23411 Mon Sep 17 00:00:00 2001 From: Jiri Date: Thu, 16 Jan 2025 15:44:58 +0100 Subject: [PATCH 19/25] Unifide testcase print --- .../openjdk/jcstress/infra/grading/XMLReportPrinter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 005c2efc..284f3dc2 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -340,7 +340,7 @@ private void emitTestReport(PrintWriter outw, Collection results, Te if (sparse) { List sorted = new ArrayList<>(results); HTMLReportPrinter.resultsOrder(sorted); - outw.println(" "); + outw.println(" " + getOpenTestcase(results, test.name())); printPropertiesPerTest(outw, test, null, sorted); printMainTestBody(outw, sorted, true); outw.println(""); @@ -356,7 +356,7 @@ private void emitTestReport(PrintWriter outw, Collection results, Te if (isTestsuiteUsed() && isStripNames()) { testName = r.getConfig().getTestVariant(false); } - outw.println(" "); + outw.println(" " + getOpenTestcase(Arrays.asList(r),testName)); printPropertiesPerTest(outw, test, r, sorted); printMainTestBody(outw, Arrays.asList(r), null); outw.println(""); @@ -368,6 +368,10 @@ private void emitTestReport(PrintWriter outw, Collection results, Te } + private static String getOpenTestcase(Collection results, String test) { + return ""; + } + private static String getTime(Collection results, boolean toNicestring) { double sum = 0; for (TestResult resul : results) { From 7d49f5c90e224f7455c9e6ef6e1984e629988e59 Mon Sep 17 00:00:00 2001 From: Jiri Date: Thu, 16 Jan 2025 17:00:38 +0100 Subject: [PATCH 20/25] changed comments proeprty to comment out non-standart elements soem of the wide spread features like nested testsuites and properties are nmot part of xsd... those will be commented out b default, and moved to fail/err if possible --- .../openjdk/jcstress/infra/grading/XMLReportPrinter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 284f3dc2..b1cb6d9b 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -92,8 +92,8 @@ private enum JunitResult { public static final String STDOUTERR_TO_FAILURE = "jcstress.report.xml.souterr2failure"; //vill validate final xmls public static final String VALIDATE = "jcstress.report.xml.validate"; - //will nto include comments (if any) - public static final String NO_COMMENTS = "jcstress.report.xml.nocomments"; + //will keep non standart (but widelyused) elements uncomment + public static final String UNCOMMENT_NONSTANDART = "jcstress.report.xml.nonstandart"; //by default both reprots are printed. By setting it to true or false, wil linclude only sparse ot full public static final String SPARSE = "jcstress.report.xml.sparse"; //true/false/null @@ -168,7 +168,7 @@ private static boolean isDuplicateProperties() { } private static boolean isNoComments() { - return System.getProperty(XMLReportPrinter.NO_COMMENTS) != null; + return System.getProperty(XMLReportPrinter.UNCOMMENT_NONSTANDART) != null; } private static String printProperty(String key, boolean value) { @@ -322,7 +322,7 @@ private void printXmlReporterProperties(PrintWriter output) { output.println(" " + printProperty(SOFT_ERROR_AS, getSoftErrorAs().toString())); output.println(" " + printProperty(HARD_ERROR_AS, getHardErrorAs().toString())); output.println(" " + printProperty(DUPLICATE_PROPERTIES, isDuplicateProperties())); - output.println(" " + printProperty(NO_COMMENTS, isNoComments())); + output.println(" " + printProperty(UNCOMMENT_NONSTANDART, isNoComments())); output.println(" " + printProperty(STDOUTERR_TO_FAILURE, isStdoutErrToFailure())); output.println(" " + printProperty(SPARSE, Objects.toString(getSparse(null)))); } From 1361ef185d5286ff1985c6e380cd2711a8d2653a Mon Sep 17 00:00:00 2001 From: Jiri Date: Sat, 18 Jan 2025 11:01:18 +0100 Subject: [PATCH 21/25] made default output xsd valid again and keeping option to enable the widely used, but not standart parts for higher readability --- .../infra/grading/XMLReportPrinter.java | 87 ++++++++++++++----- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index b1cb6d9b..4a607628 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -167,7 +167,7 @@ private static boolean isDuplicateProperties() { return System.getProperty(XMLReportPrinter.DUPLICATE_PROPERTIES) != null; } - private static boolean isNoComments() { + private static boolean isXsdLenient() { return System.getProperty(XMLReportPrinter.UNCOMMENT_NONSTANDART) != null; } @@ -238,22 +238,19 @@ public void work() throws FileNotFoundException { PrintWriter output = new PrintWriter(filePath); output.println(""); - String header = printTestSuiteHeader(byName, "jcstress"); - output.println(header); - { - - output.println(" "); - output.print(getBaseProperties(byName)); - //FIXME print all properties, if 7903889 got ever implemented - printXmlReporterProperties(output); - output.println(" "); + output.println(""); + if (isMainTestsuite()) { + String header = printTestSuiteHeader(byName, "jcstress"); + output.println(" " + header); + printMainproperties(output, byName); } - byName = ReportUtils.mergedByName(collector.getTestResults()); Collections.sort(byName, Comparator.comparing(TestResult::getName)); - emitTestReports(ReportUtils.byName(collector.getTestResults()), output); - output.println(""); + if (isMainTestsuite()) { + output.println(" "); + } + output.println(""); output.flush(); output.close(); if (isValidate()) { @@ -261,6 +258,26 @@ public void work() throws FileNotFoundException { } } + private void printMainproperties(PrintWriter output, List byName) { + output.println(" "); + output.print(getBaseProperties(byName)); + //FIXME print all properties, if 7903889 got ever implemented + printXmlReporterProperties(output); + output.println(" "); + } + + private boolean isMainTestsuite() { + if (sparse) { + return true; + } else { + if (!isTestsuiteUsed()) { + return true; + } else { + return isXsdLenient(); + } + } + } + private static String printTestSuiteHeader(List results, String name) { Map> reordered = countJunitResults(results); String hostname = "localhost"; @@ -269,12 +286,18 @@ private static String printTestSuiteHeader(List results, String name } catch (Exception ex) { //no interest } - return "\n"; + passed = ""; + } + return prefix + ""; @@ -322,7 +345,7 @@ private void printXmlReporterProperties(PrintWriter output) { output.println(" " + printProperty(SOFT_ERROR_AS, getSoftErrorAs().toString())); output.println(" " + printProperty(HARD_ERROR_AS, getHardErrorAs().toString())); output.println(" " + printProperty(DUPLICATE_PROPERTIES, isDuplicateProperties())); - output.println(" " + printProperty(UNCOMMENT_NONSTANDART, isNoComments())); + output.println(" " + printProperty(UNCOMMENT_NONSTANDART, isXsdLenient())); output.println(" " + printProperty(STDOUTERR_TO_FAILURE, isStdoutErrToFailure())); output.println(" " + printProperty(SPARSE, Objects.toString(getSparse(null)))); } @@ -341,7 +364,9 @@ private void emitTestReport(PrintWriter outw, Collection results, Te List sorted = new ArrayList<>(results); HTMLReportPrinter.resultsOrder(sorted); outw.println(" " + getOpenTestcase(results, test.name())); - printPropertiesPerTest(outw, test, null, sorted); + if (isXsdLenient()) { + printPropertiesPerTest(outw, test, null, sorted); + } printMainTestBody(outw, sorted, true); outw.println(""); } else { @@ -350,14 +375,19 @@ private void emitTestReport(PrintWriter outw, Collection results, Te if (isTestsuiteUsed()) { String header = printTestSuiteHeader(sorted, suiteCandidate); outw.println(header); + if (!isMainTestsuite()) { + printMainproperties(outw, sorted); + } } for (TestResult r : sorted) { String testName = r.getConfig().toDetailedTest(false); if (isTestsuiteUsed() && isStripNames()) { testName = r.getConfig().getTestVariant(false); } - outw.println(" " + getOpenTestcase(Arrays.asList(r),testName)); - printPropertiesPerTest(outw, test, r, sorted); + outw.println(" " + getOpenTestcase(Arrays.asList(r), testName)); + if (isXsdLenient()) { + printPropertiesPerTest(outw, test, r, sorted); + } printMainTestBody(outw, Arrays.asList(r), null); outw.println(""); } @@ -384,7 +414,7 @@ private static String getTime(Collection results, boolean toNicestri return String.valueOf(sum); } } else { - return ""+sum; + return "" + sum; } } @@ -459,7 +489,15 @@ private static void printStatusElement(PrintWriter outw, List result if (junitResult == JunitResult.failure || junitResult == JunitResult.error) { outw.println("<" + junitResult + "> result outw.println("]]>"); } if (junitResult == JunitResult.skipped) { - outw.println(" "); + if (isXsdLenient()) { + outw.println(" "); + } else { + outw.println(" "); + outw.println(" "); + } } } From 5c77eae889372478317be4276e763d976b0f33f2 Mon Sep 17 00:00:00 2001 From: Jiri Date: Sat, 25 Jan 2025 12:43:37 +0100 Subject: [PATCH 22/25] Fixed few noted comments/variables --- .../jcstress/infra/grading/XMLReportPrinter.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 4a607628..0ab80b00 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -76,9 +76,9 @@ private enum JunitResult { } //how to deal with sofrt errors like api mishmash or similar - public static final String SOFT_ERROR_AS = "jcstress.report.xml.softErrorAs"; //pass/fail/skip defaults to skip + public static final String SOFT_ERROR_AS = "jcstress.report.xml.softErrorAs"; //pass/fail/skip/error defaults to skip //how to deal with hard errors. Those may be timout, but also segfaulting vm - public static final String HARD_ERROR_AS = "jcstress.report.xml.hardErrorAs"; //pass/fail defaults to fail + public static final String HARD_ERROR_AS = "jcstress.report.xml.hardErrorAs"; //pass/fail/skip/error defaults to fail //only for full (non-saprse) output, will wrap each family by its public static final String USE_TESTSUITES = "jcstress.report.xml.sparse.testsuites"; //in case of sued testsuiotes, will not replicate the name of suite in test name. @@ -90,11 +90,11 @@ private enum JunitResult { //this is for tools,m which do nto show stdout/err properly //also it is saving a bit of space, but is loosing the granularity public static final String STDOUTERR_TO_FAILURE = "jcstress.report.xml.souterr2failure"; - //vill validate final xmls + //will validate final xmls public static final String VALIDATE = "jcstress.report.xml.validate"; //will keep non standart (but widelyused) elements uncomment public static final String UNCOMMENT_NONSTANDART = "jcstress.report.xml.nonstandart"; - //by default both reprots are printed. By setting it to true or false, wil linclude only sparse ot full + //by default both reports are printed. By setting it to true or false, will include only sparse ot full public static final String SPARSE = "jcstress.report.xml.sparse"; //true/false/null private final String resultDir; @@ -637,7 +637,7 @@ private void validByXsd(String xml) throws ParserConfigurationException, SAXExce public static JunitResult testsToJunitResult(Collection results) { boolean hadError = false; - int coutSkipped = 0; + int countSkipped = 0; for (TestResult result : results) { if (testToJunitResult(result) == JunitResult.failure) { //if there was failure in sub set, return whole group as failure @@ -647,7 +647,7 @@ public static JunitResult testsToJunitResult(Collection results) { hadError = true; } if (testToJunitResult(result) == JunitResult.skipped) { - coutSkipped++; + countSkipped++; } } //no failure, bute errors presented @@ -655,7 +655,7 @@ public static JunitResult testsToJunitResult(Collection results) { return JunitResult.error; } //no failure, no error, was all skipped? - if (coutSkipped == results.size()) { + if (countSkipped == results.size()) { return JunitResult.skipped; } return JunitResult.pass; From 80f3586bf9bafe8a44f8283e90f00598e0ec3220 Mon Sep 17 00:00:00 2001 From: Jiri Date: Sat, 25 Jan 2025 13:28:35 +0100 Subject: [PATCH 23/25] Revert "FIXME Added fast fake results generator REMOVE IT" This reverts commit 0d758a79012dcb11f2a98b74a326b58809a97599. The junit result looks good. Now testing on real data --- .../jcstress/infra/collectors/TestResult.java | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java index cbd7eca9..ed4c2b79 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java @@ -39,7 +39,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Random; /** * @author Aleksey Shipilev (aleksey.shipilev@oracle.com) @@ -55,29 +54,15 @@ public class TestResult implements Serializable { private final List vmErr; private transient TestGrading grading; - private static final Random r = new Random(); - private final int failer; - public TestResult(Status status) { - failer = r.nextInt(7); - switch (failer) { - case 0: this.status=Status.TEST_ERROR; break; - case 1: this.status=Status.NORMAL; break; - case 2: this.status=Status.CHECK_TEST_ERROR; break; - case 3: this.status=Status.API_MISMATCH; break; - case 4: this.status=Status.TIMEOUT_ERROR; break; - case 5: this.status=Status.VM_ERROR; break; - default: this.status = status; - } + this.status = status; this.states = new Counter<>(); this.messages = new ArrayList<>(); this.vmOut = new ArrayList<>(); this.vmErr = new ArrayList<>(); - fakeOutputs(); } public TestResult(DataInputStream dis) throws IOException { - failer=10; status = Status.values()[dis.readInt()]; states = new Counter<>(dis); messages = new ArrayList<>(); @@ -101,16 +86,6 @@ public TestResult(DataInputStream dis) throws IOException { vmErr.add(dis.readUTF()); } } - fakeOutputs(); - } - - private void fakeOutputs() { - messages.add("messages1"); - messages.add("messages2"); - vmOut.add("out1"); - vmOut.add("out2"); - vmErr.add("err1"); - vmErr.add("err2"); } public void write(DataOutputStream dos) throws IOException { From 96b4e81a8a9bc18efd4a0d09ccd32c737013256e Mon Sep 17 00:00:00 2001 From: Jiri Date: Sat, 25 Jan 2025 17:04:32 +0100 Subject: [PATCH 24/25] Started migration to proepr DOM --- .../infra/grading/XMLReportPrinter.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 0ab80b00..618d1c34 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -34,7 +34,10 @@ import org.openjdk.jcstress.os.SchedulingClass; import org.openjdk.jcstress.util.Multimap; import org.openjdk.jcstress.vm.CompileMode; +import org.w3c.dom.CDATASection; +import org.w3c.dom.Comment; import org.w3c.dom.Document; +import org.w3c.dom.Element; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -42,6 +45,12 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -50,6 +59,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.InetAddress; @@ -679,4 +689,41 @@ public static JunitResult testToJunitResult(TestResult result) { } } + public static void main(String... args) throws ParserConfigurationException, TransformerException { + DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + + Document doc = docBuilder.newDocument(); + Comment comment1 = doc.createComment("passed 4"); + doc.appendChild(comment1); + Element testsuite = doc.createElement("testsuite"); + testsuite.setAttribute("failed", "5"); + testsuite.setAttribute("total", "10"); + testsuite.setAttribute("error", "8"); + testsuite.setAttribute("skipped", "9"); + doc.appendChild(testsuite); + Comment comment2 = doc.createComment("passed 5"); + testsuite.appendChild(comment2); + Element testcase = doc.createElement("testcase"); + testsuite.appendChild(testcase); + testcase.setAttribute("classname", "jcstress"); + testcase.setAttribute("name", "org.openjdk.jcstress.tests.copy.manual.objects.plain.StringTest"); + Element failure = doc.createElement("failure"); + CDATASection cdataSection = doc.createCDATASection("more\nlines\nhere"); + failure.appendChild(cdataSection); + testcase.appendChild(failure); + writeXml(doc, System.out); + + } + + // write doc to output stream + private static void writeXml(Document doc, OutputStream output) throws TransformerException { + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + DOMSource source = new DOMSource(doc); + StreamResult result = new StreamResult(output); + transformer.transform(source, result); + } + } From 910b6ba53e7bc911f815bdfb7490d5243975356a Mon Sep 17 00:00:00 2001 From: Jiri Vanek Date: Mon, 27 Jan 2025 10:46:29 +0100 Subject: [PATCH 25/25] Dropped the dom-based generation Without stream the serilializer was unnecessarily big --- .../infra/grading/XMLReportPrinter.java | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java index 618d1c34..bf2ca6cb 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -34,10 +34,7 @@ import org.openjdk.jcstress.os.SchedulingClass; import org.openjdk.jcstress.util.Multimap; import org.openjdk.jcstress.vm.CompileMode; -import org.w3c.dom.CDATASection; -import org.w3c.dom.Comment; import org.w3c.dom.Document; -import org.w3c.dom.Element; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -45,12 +42,6 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -59,7 +50,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.InetAddress; @@ -688,42 +678,4 @@ public static JunitResult testToJunitResult(TestResult result) { throw new IllegalStateException("Illegal status: " + result.status()); } } - - public static void main(String... args) throws ParserConfigurationException, TransformerException { - DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); - - Document doc = docBuilder.newDocument(); - Comment comment1 = doc.createComment("passed 4"); - doc.appendChild(comment1); - Element testsuite = doc.createElement("testsuite"); - testsuite.setAttribute("failed", "5"); - testsuite.setAttribute("total", "10"); - testsuite.setAttribute("error", "8"); - testsuite.setAttribute("skipped", "9"); - doc.appendChild(testsuite); - Comment comment2 = doc.createComment("passed 5"); - testsuite.appendChild(comment2); - Element testcase = doc.createElement("testcase"); - testsuite.appendChild(testcase); - testcase.setAttribute("classname", "jcstress"); - testcase.setAttribute("name", "org.openjdk.jcstress.tests.copy.manual.objects.plain.StringTest"); - Element failure = doc.createElement("failure"); - CDATASection cdataSection = doc.createCDATASection("more\nlines\nhere"); - failure.appendChild(cdataSection); - testcase.appendChild(failure); - writeXml(doc, System.out); - - } - - // write doc to output stream - private static void writeXml(Document doc, OutputStream output) throws TransformerException { - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(output); - transformer.transform(source, result); - } - }
" + c.expect + "" + c.description + "" + c.count + "0