Skip to content

Commit

Permalink
Fix panic in flakeguard runner
Browse files Browse the repository at this point in the history 
https://github.com/smartcontractkit/chainlink/actions/runs/12200639955/job/34037331531
  • Loading branch information
@lukaszcl
lukaszcl committed Dec 6, 2024
1 parent eb1cdf6 commit 04f007a
Showing 1 changed file with 69 additions and 49 deletions.
118 changes: 69 additions & 49 deletions tools/flakeguard/runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
r.rawOutputs[p] = &bytes.Buffer{}
}
separator := strings.Repeat("-", 80)
r.rawOutputs[p].WriteString(fmt.Sprintf("Run %d%s\n", i+1, separator))
r.rawOutputs[p].WriteString(fmt.Sprintf("Run %d\n%s\n", i+1, separator))
}
jsonFilePath, passed, err := r.runTests(p)
if err != nil {
Expand All @@ -83,7 +83,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
}, nil
}

// RawOutput retrieves the raw output from the test runs, if CollectRawOutput enabled.
// RawOutputs retrieves the raw output from the test runs, if CollectRawOutput enabled.
// packageName : raw output
func (r *Runner) RawOutputs() map[string]*bytes.Buffer {
return r.rawOutputs
Expand Down Expand Up @@ -120,7 +120,6 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
selectPattern := strings.Join(r.SelectTests, "$|^")
args = append(args, fmt.Sprintf("-run=^%s$", selectPattern))
}
args = append(args, "2>/dev/null")

if r.Verbose {
log.Printf("Running command: go %s\n", strings.Join(args, " "))
Expand Down Expand Up @@ -249,7 +248,6 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
result = testDetails[key]
}

// TODO: This is a bit of a logical mess, probably worth a refactor
if entryLine.Output != "" {
if panicDetectionMode || raceDetectionMode { // currently collecting panic or race output
detectedEntries = append(detectedEntries, entryLine)
Expand Down Expand Up @@ -281,22 +279,32 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
return nil, err
}
panicTestKey := fmt.Sprintf("%s/%s", entryLine.Package, panicTest)
testDetails[panicTestKey].Panic = true
testDetails[panicTestKey].Timeout = timeout
testDetails[panicTestKey].Failures++
testDetails[panicTestKey].Runs++
// TODO: durations and panics are weird in the same way as Runs: lots of double-counting
// duration, err := time.ParseDuration(strconv.FormatFloat(entryLine.Elapsed, 'f', -1, 64) + "s")
// if err != nil {
// return nil, fmt.Errorf("failed to parse duration: %w", err)
// }
// testDetails[panicTestKey].Durations = append(testDetails[panicTestKey].Durations, duration)
testDetails[panicTestKey].Outputs = append(testDetails[panicTestKey].Outputs, entryLine.Output)

// Ensure the test exists in testDetails
result, exists := testDetails[panicTestKey]
if !exists {
// Create a new TestResult if it doesn't exist
result = &reports.TestResult{
TestName: panicTest,
TestPackage: entryLine.Package,
PassRatio: 0,
Outputs: []string{},
PackageOutputs: []string{},
}
testDetails[panicTestKey] = result
}

result.Panic = true
result.Timeout = timeout
result.Failures++
result.Runs++

// Handle outputs
for _, entry := range detectedEntries {
if entry.Test == "" {
testDetails[panicTestKey].PackageOutputs = append(testDetails[panicTestKey].PackageOutputs, entry.Output)
result.PackageOutputs = append(result.PackageOutputs, entry.Output)
} else {
testDetails[panicTestKey].Outputs = append(testDetails[panicTestKey].Outputs, entry.Output)
result.Outputs = append(result.Outputs, entry.Output)
}
}
} else if raceDetectionMode {
Expand All @@ -305,21 +313,31 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
return nil, err
}
raceTestKey := fmt.Sprintf("%s/%s", entryLine.Package, raceTest)
testDetails[raceTestKey].Race = true
testDetails[raceTestKey].Failures++
testDetails[raceTestKey].Runs++
// TODO: durations and races are weird in the same way as Runs: lots of double-counting
// duration, err := time.ParseDuration(strconv.FormatFloat(entryLine.Elapsed, 'f', -1, 64) + "s")
// if err != nil {
// return nil, fmt.Errorf("failed to parse duration: %w", err)
// }
// testDetails[raceTestKey].Durations = append(testDetails[raceTestKey].Durations, duration)
testDetails[raceTestKey].Outputs = append(testDetails[raceTestKey].Outputs, entryLine.Output)

// Ensure the test exists in testDetails
result, exists := testDetails[raceTestKey]
if !exists {
// Create a new TestResult if it doesn't exist
result = &reports.TestResult{
TestName: raceTest,
TestPackage: entryLine.Package,
PassRatio: 0,
Outputs: []string{},
PackageOutputs: []string{},
}
testDetails[raceTestKey] = result
}

result.Race = true
result.Failures++
result.Runs++

// Handle outputs
for _, entry := range detectedEntries {
if entry.Test == "" {
testDetails[raceTestKey].PackageOutputs = append(testDetails[raceTestKey].PackageOutputs, entry.Output)
result.PackageOutputs = append(result.PackageOutputs, entry.Output)
} else {
testDetails[raceTestKey].Outputs = append(testDetails[raceTestKey].Outputs, entry.Output)
result.Outputs = append(result.Outputs, entry.Output)
}
}
}
Expand Down Expand Up @@ -383,28 +401,28 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
if parentTestResult, exists := testDetails[parentTestKey]; exists {
if parentTestResult.Panic {
for _, subTest := range subTests {
subTestKey := fmt.Sprintf("%s/%s", parentTestKey, subTest)
if _, exists := testDetails[subTestKey]; exists {
if testDetails[subTestKey].Failures > 0 { // If the parent test panicked, any of its subtests that failed could be the culprit
testDetails[subTestKey].Panic = true
testDetails[subTestKey].Outputs = append(testDetails[subTestKey].Outputs, "Panic in parent test")
subTestKey := fmt.Sprintf("%s/%s", parentTestResult.TestPackage, subTest)
if subTestResult, exists := testDetails[subTestKey]; exists {
if subTestResult.Failures > 0 { // If the parent test panicked, any of its subtests that failed could be the culprit
subTestResult.Panic = true
subTestResult.Outputs = append(subTestResult.Outputs, "Panic in parent test")
}
} else {
fmt.Printf("WARN: expected to fine subtest '%s' inside parent test '%s', but not found\n", parentTestKey, subTestKey)
log.Printf("WARN: expected to find subtest '%s' inside parent test '%s', but not found\n", subTestKey, parentTestKey)
}
}
}
} else {
fmt.Printf("WARN: expected to find parent test '%s' for sub tests, but not found\n", parentTestKey)
log.Printf("WARN: expected to find parent test '%s' for subtests, but not found\n", parentTestKey)
}
}
for _, result := range testDetails {
if result.Runs > expectedRuns { // Panics can introduce double-counting test failures, this is a hacky correction for it
if result.Runs > expectedRuns { // Panics can introduce double-counting test failures, this is a correction for it
if result.Panic {
result.Failures = expectedRuns
result.Runs = expectedRuns
} else {
fmt.Printf("WARN: '%s' has %d test runs, exceeding expected amount of %d in an unexpected way, this may be due to oddly presented panics\n", result.TestName, result.Runs, expectedRuns)
log.Printf("WARN: '%s' has %d test runs, exceeding expected amount of %d; this may be due to unexpected panics\n", result.TestName, result.Runs, expectedRuns)
}
}
// If a package panicked, all tests in that package will be marked as panicking
Expand All @@ -420,8 +438,7 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
return results, nil
}

// properly attributes panics to the test that caused them
// Go JSON output gets confused, especially when tests are run in parallel
// attributePanicToTest properly attributes panics to the test that caused them.
func attributePanicToTest(panicPackage string, panicEntries []entry) (test string, timeout bool, err error) {
regexSanitizePanicPackage := filepath.Base(panicPackage)
panicAttributionRe := regexp.MustCompile(fmt.Sprintf(`%s\.(Test[^\.\(]+)`, regexSanitizePanicPackage))
Expand All @@ -430,31 +447,33 @@ func attributePanicToTest(panicPackage string, panicEntries []entry) (test strin
for _, entry := range panicEntries {
entriesOutputs = append(entriesOutputs, entry.Output)
if matches := panicAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
return matches[1], false, nil
testName := strings.TrimSpace(matches[1])
return testName, false, nil
}
if matches := timeoutAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
return matches[1], true, nil
testName := strings.TrimSpace(matches[1])
return testName, true, nil
}
}
return "", false, fmt.Errorf("failed to attribute panic to test, using regex %s on these strings:\n%s", panicAttributionRe.String(), strings.Join(entriesOutputs, ""))
}

// properly attributes races to the test that caused them
// Go JSON output gets confused, especially when tests are run in parallel
// attributeRaceToTest properly attributes races to the test that caused them.
func attributeRaceToTest(racePackage string, raceEntries []entry) (string, error) {
regexSanitizeRacePackage := filepath.Base(racePackage)
raceAttributionRe := regexp.MustCompile(fmt.Sprintf(`%s\.(Test[^\.\(]+)`, regexSanitizeRacePackage))
entriesOutputs := []string{}
for _, entry := range raceEntries {
entriesOutputs = append(entriesOutputs, entry.Output)
if matches := raceAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
return matches[1], nil
testName := strings.TrimSpace(matches[1])
return testName, nil
}
}
return "", fmt.Errorf("failed to attribute race to test, using regex: %s on these strings:\n%s", raceAttributionRe.String(), strings.Join(entriesOutputs, ""))
}

// parseSubTest checks if a test name is a subtest and returns the parent and sub names
// parseSubTest checks if a test name is a subtest and returns the parent and sub names.
func parseSubTest(testName string) (parentTestName, subTestName string) {
parts := strings.SplitN(testName, "/", 2)
if len(parts) == 1 {
Expand All @@ -463,7 +482,7 @@ func parseSubTest(testName string) (parentTestName, subTestName string) {
return parts[0], parts[1]
}

// prettyProjectPath returns the project path formatted for pretty printing in results
// prettyProjectPath returns the project path formatted for pretty printing in results.
func prettyProjectPath(projectPath string) (string, error) {
// Walk up the directory structure to find go.mod
absPath, err := filepath.Abs(projectPath)
Expand Down Expand Up @@ -493,8 +512,9 @@ func prettyProjectPath(projectPath string) (string, error) {
for _, line := range strings.Split(string(goModData), "\n") {
if strings.HasPrefix(line, "module ") {
goProject := strings.TrimSpace(strings.TrimPrefix(line, "module "))
projectPath = strings.TrimPrefix(projectPath, goProject)
return filepath.Join(goProject, projectPath), nil
relativePath := strings.TrimPrefix(projectPath, dir)
relativePath = strings.TrimLeft(relativePath, string(os.PathSeparator))
return filepath.Join(goProject, relativePath), nil
}
}

Expand Down

0 comments on commit 04f007a

Please sign in to comment.