Fix missing outputs

smartcontractkit · Dec 6, 2024 · 1540a1b · 1540a1b
1 parent 084ceff
commit 1540a1b
Show file tree

Hide file tree

Showing 2 changed files with 164 additions and 68 deletions.
diff --git a/tools/flakeguard/cmd/report.go b/tools/flakeguard/cmd/report.go
@@ -83,19 +83,51 @@ var ReportCmd = &cobra.Command{
 			fmt.Println("Test results mapped to code owners successfully.")
 		}
 
-		// Exclude outputs and package outputs from the aggregated report of all tests
-		for i := range aggregatedReport.Results {
-			aggregatedReport.Results[i].Outputs = nil
-			aggregatedReport.Results[i].PackageOutputs = nil
-		}
-
 		// Create output directory if it doesn't exist
 		outputDir := reportOutputPath
 		if err := fs.MkdirAll(outputDir, 0755); err != nil {
 			return fmt.Errorf("error creating output directory: %w", err)
 		}
 
-		// Save the aggregated report (all tests)
+		// Filter failed tests (PassRatio < maxPassRatio and not skipped)
+		s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
+		s.Suffix = " Filtering failed tests..."
+		s.Start()
+
+		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
+			return !tr.Skipped && tr.PassRatio < reportMaxPassRatio
+		})
+		s.Stop()
+		fmt.Println("Failed tests filtered successfully.")
+
+		// Create a new report for failed tests with logs
+		failedReportWithLogs := &reports.TestReport{
+			GoProject:     aggregatedReport.GoProject,
+			TestRunCount:  aggregatedReport.TestRunCount,
+			RaceDetection: aggregatedReport.RaceDetection,
+			ExcludedTests: aggregatedReport.ExcludedTests,
+			SelectedTests: aggregatedReport.SelectedTests,
+			Results:       failedTests,
+		}
+
+		// Save the failed tests report with logs
+		failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
+		if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
+			return fmt.Errorf("error saving failed tests report with logs: %w", err)
+		}
+		fmt.Printf("Failed tests report with logs saved to %s\n", failedTestsReportWithLogsPath)
+
+		// Set Outputs and PackageOutputs to nil for reports without logs
+		for i := range aggregatedReport.Results {
+			aggregatedReport.Results[i].Outputs = nil
+			aggregatedReport.Results[i].PackageOutputs = nil
+		}
+		for i := range failedTests {
+			failedTests[i].Outputs = nil
+			failedTests[i].PackageOutputs = nil
+		}
+
+		// Save the aggregated report (all tests) without logs
 		allTestsReportPath := filepath.Join(outputDir, "all-test-results.json")
 		if err := reports.SaveReport(fs, allTestsReportPath, *aggregatedReport); err != nil {
 			return fmt.Errorf("error saving all tests report: %w", err)
@@ -168,18 +200,7 @@ var ReportCmd = &cobra.Command{
 			fmt.Println("PR comment markdown generated successfully.")
 		}
 
-		// Filter failed tests (PassRatio < maxPassRatio and not skipped)
-		s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
-		s.Suffix = " Filtering failed tests..."
-		s.Start()
-
-		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
-			return !tr.Skipped && tr.PassRatio < reportMaxPassRatio
-		})
-		s.Stop()
-		fmt.Println("Failed tests filtered successfully.")
-
-		// Create a new report for failed tests
+		// Create a new report for failed tests without logs
 		failedReportNoLogs := &reports.TestReport{
 			GoProject:     aggregatedReport.GoProject,
 			TestRunCount:  aggregatedReport.TestRunCount,
@@ -189,37 +210,13 @@ var ReportCmd = &cobra.Command{
 			Results:       failedTests,
 		}
 
-		// Save the failed tests report with no logs
+		// Save the failed tests report without logs
 		failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-results.json")
 		if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportNoLogs); err != nil {
-			return fmt.Errorf("error saving failed tests report: %w", err)
+			return fmt.Errorf("error saving failed tests report without logs: %w", err)
 		}
 		fmt.Printf("Failed tests report without logs saved to %s\n", failedTestsReportNoLogsPath)
 
-		// Retrieve outputs and package outputs for failed tests
-		for i := range failedTests {
-			// Retrieve outputs and package outputs from original reports
-			failedTests[i].Outputs = getOriginalOutputs(testReports, failedTests[i].TestName, failedTests[i].TestPackage)
-			failedTests[i].PackageOutputs = getOriginalPackageOutputs(testReports, failedTests[i].TestName, failedTests[i].TestPackage)
-		}
-
-		// Create a new report for failed tests
-		failedReportWithLogs := &reports.TestReport{
-			GoProject:     aggregatedReport.GoProject,
-			TestRunCount:  aggregatedReport.TestRunCount,
-			RaceDetection: aggregatedReport.RaceDetection,
-			ExcludedTests: aggregatedReport.ExcludedTests,
-			SelectedTests: aggregatedReport.SelectedTests,
-			Results:       failedTests,
-		}
-
-		// Save the failed tests report
-		failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
-		if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
-			return fmt.Errorf("error saving failed tests report: %w", err)
-		}
-		fmt.Printf("Failed tests report with logs saved to %s\n", failedTestsReportWithLogsPath)
-
 		fmt.Printf("Reports generated at: %s\n", reportOutputPath)
 
 		return nil
@@ -288,26 +285,3 @@ func generateAllTestsSummaryJSON(report *reports.TestReport, outputPath string,
 
 	return nil
 }
-
-// Helper functions to retrieve original outputs and package outputs
-func getOriginalOutputs(reports []*reports.TestReport, testName, testPackage string) []string {
-	for _, report := range reports {
-		for _, result := range report.Results {
-			if result.TestName == testName && result.TestPackage == testPackage {
-				return result.Outputs
-			}
-		}
-	}
-	return nil
-}
-
-func getOriginalPackageOutputs(reports []*reports.TestReport, testName, testPackage string) []string {
-	for _, report := range reports {
-		for _, result := range report.Results {
-			if result.TestName == testName && result.TestPackage == testPackage {
-				return result.PackageOutputs
-			}
-		}
-	}
-	return nil
-}
diff --git a/tools/flakeguard/reports/data_test.go b/tools/flakeguard/reports/data_test.go
@@ -340,6 +340,128 @@ func TestAggregate(t *testing.T) {
 	}
 }
 
+func TestAggregateOutputs(t *testing.T) {
+	report1 := &TestReport{
+		GoProject:    "ProjectX",
+		TestRunCount: 1,
+		Results: []TestResult{
+			{
+				TestName:       "TestOutput",
+				TestPackage:    "pkg1",
+				Runs:           1,
+				Successes:      1,
+				Outputs:        []string{"Output from report1 test run"},
+				PackageOutputs: []string{"Package output from report1"},
+			},
+		},
+	}
+
+	report2 := &TestReport{
+		GoProject:    "ProjectX",
+		TestRunCount: 1,
+		Results: []TestResult{
+			{
+				TestName:       "TestOutput",
+				TestPackage:    "pkg1",
+				Runs:           1,
+				Successes:      1,
+				Outputs:        []string{"Output from report2 test run"},
+				PackageOutputs: []string{"Package output from report2"},
+			},
+		},
+	}
+
+	aggregatedReport, err := Aggregate(report1, report2)
+	if err != nil {
+		t.Fatalf("Error aggregating reports: %v", err)
+	}
+
+	if len(aggregatedReport.Results) != 1 {
+		t.Fatalf("Expected 1 result, got %d", len(aggregatedReport.Results))
+	}
+
+	result := aggregatedReport.Results[0]
+
+	// Expected outputs
+	expectedOutputs := []string{
+		"Output from report1 test run",
+		"Output from report2 test run",
+	}
+	expectedPackageOutputs := []string{
+		"Package output from report1",
+		"Package output from report2",
+	}
+
+	if !reflect.DeepEqual(result.Outputs, expectedOutputs) {
+		t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.Outputs)
+	}
+
+	if !reflect.DeepEqual(result.PackageOutputs, expectedPackageOutputs) {
+		t.Errorf("Expected PackageOutputs %v, got %v", expectedPackageOutputs, result.PackageOutputs)
+	}
+}
+
+func TestAggregateIdenticalOutputs(t *testing.T) {
+	report1 := &TestReport{
+		GoProject:    "ProjectX",
+		TestRunCount: 1,
+		Results: []TestResult{
+			{
+				TestName:       "TestIdenticalOutput",
+				TestPackage:    "pkg1",
+				Runs:           1,
+				Successes:      1,
+				Outputs:        []string{"Identical output"},
+				PackageOutputs: []string{"Identical package output"},
+			},
+		},
+	}
+
+	report2 := &TestReport{
+		GoProject:    "ProjectX",
+		TestRunCount: 1,
+		Results: []TestResult{
+			{
+				TestName:       "TestIdenticalOutput",
+				TestPackage:    "pkg1",
+				Runs:           1,
+				Successes:      1,
+				Outputs:        []string{"Identical output"},
+				PackageOutputs: []string{"Identical package output"},
+			},
+		},
+	}
+
+	aggregatedReport, err := Aggregate(report1, report2)
+	if err != nil {
+		t.Fatalf("Error aggregating reports: %v", err)
+	}
+
+	if len(aggregatedReport.Results) != 1 {
+		t.Fatalf("Expected 1 result, got %d", len(aggregatedReport.Results))
+	}
+
+	result := aggregatedReport.Results[0]
+
+	// Expected outputs
+	expectedOutputs := []string{
+		"Identical output",
+		"Identical output",
+	}
+	expectedPackageOutputs := []string{
+		"Identical package output",
+		"Identical package output",
+	}
+
+	if !reflect.DeepEqual(result.Outputs, expectedOutputs) {
+		t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.Outputs)
+	}
+
+	if !reflect.DeepEqual(result.PackageOutputs, expectedPackageOutputs) {
+		t.Errorf("Expected PackageOutputs %v, got %v", expectedPackageOutputs, result.PackageOutputs)
+	}
+}
+
 // TestMergeTestResults tests the mergeTestResults function.
 func TestMergeTestResults(t *testing.T) {
 	a := TestResult{