Skip to content

Commit

Permalink
[ADXT-716] Test washer failing wash known flaky unit test (#32720)
Browse files Browse the repository at this point in the history
  • Loading branch information
@KevinFairise2 @mwdd146980
KevinFairise2 authored and mwdd146980 committed Jan 10, 2025
1 parent f4ddcd7 commit f509427
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 0 deletions.
4 changes: 4 additions & 0 deletions tasks/testwasher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ def parse_test_results(self, module_path: str) -> tuple[dict, dict]:
if test_result["Action"] == "success":
if test_result["Test"] in failing_tests[test_result["Package"]]:
failing_tests[test_result["Package"]].remove(test_result["Test"])
# Tests that have a go routine that panicked does not have an Action field with the result of the test, let's try to catch them from their Output
if "Output" in test_result and "panic:" in test_result["Output"]:
failing_tests[test_result["Package"]].add(test_result["Test"])

if "Output" in test_result and self.flaky_test_indicator in test_result["Output"]:
flaky_marked_tests[test_result["Package"]].add(test_result["Test"])
return failing_tests, flaky_marked_tests
Expand Down
2 changes: 2 additions & 0 deletions tasks/unit_tests/testdata/flakes_4.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pkg/serverless/trace:
- TestLoadConfigShouldBeFast
23 changes: 23 additions & 0 deletions tasks/unit_tests/testdata/test_output_failure_flaky_panic.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{"Time":"2025-01-07T10:50:32.435732+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace"}
{"Time":"2025-01-07T10:50:32.435851+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan"}
{"Time":"2025-01-07T10:50:32.435878+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"testing: warning: no tests to run\n"}
{"Time":"2025-01-07T10:50:32.435908+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"PASS\n"}
{"Time":"2025-01-07T10:50:32.435913+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"ok \tgithub.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan\t(cached) [no tests to run]\n"}
{"Time":"2025-01-07T10:50:32.435919+01:00","Action":"pass","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Elapsed":0}
{"Time":"2025-01-07T10:50:32.435982+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation"}
{"Time":"2025-01-07T10:50:32.436028+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"testing: warning: no tests to run\n"}
{"Time":"2025-01-07T10:50:32.436035+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"PASS\n"}
{"Time":"2025-01-07T10:50:32.436041+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"ok \tgithub.com/DataDog/datadog-agent/pkg/serverless/trace/propagation\t(cached) [no tests to run]\n"}
{"Time":"2025-01-07T10:50:32.436046+01:00","Action":"pass","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Elapsed":0}
{"Time":"2025-01-07T10:50:33.460941+01:00","Action":"run","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast"}
{"Time":"2025-01-07T10:50:33.461002+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"=== RUN TestLoadConfigShouldBeFast\n"}
{"Time":"2025-01-07T10:50:33.461097+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":" trace_test.go:104: flakytest: this is a known flaky test\n"}
{"Time":"2025-01-07T10:50:33.464674+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"panic: toto\n"}
{"Time":"2025-01-07T10:50:33.464694+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\n"}
{"Time":"2025-01-07T10:50:33.464697+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"goroutine 213 [running]:\n"}
{"Time":"2025-01-07T10:50:33.464713+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"github.com/DataDog/datadog-agent/pkg/serverless/trace.TestLoadConfigShouldBeFast.func1()\n"}
{"Time":"2025-01-07T10:50:33.464726+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\t/Users/kevin.fairise/go/src/github.com/DataDog/datadog-agent/pkg/serverless/trace/trace_test.go:106 +0x2c\n"}
{"Time":"2025-01-07T10:50:33.464728+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"created by github.com/DataDog/datadog-agent/pkg/serverless/trace.TestLoadConfigShouldBeFast in goroutine 212\n"}
{"Time":"2025-01-07T10:50:33.46473+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\t/Users/kevin.fairise/go/src/github.com/DataDog/datadog-agent/pkg/serverless/trace/trace_test.go:105 +0x44\n"}
{"Time":"2025-01-07T10:50:33.465562+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Output":"FAIL\tgithub.com/DataDog/datadog-agent/pkg/serverless/trace\t1.029s\n"}
{"Time":"2025-01-07T10:50:33.465624+01:00","Action":"fail","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Elapsed":1.03}
22 changes: 22 additions & 0 deletions tasks/unit_tests/testdata/test_output_failure_panic.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{"Time":"2025-01-07T10:50:32.435732+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace"}
{"Time":"2025-01-07T10:50:32.435851+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan"}
{"Time":"2025-01-07T10:50:32.435878+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"testing: warning: no tests to run\n"}
{"Time":"2025-01-07T10:50:32.435908+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"PASS\n"}
{"Time":"2025-01-07T10:50:32.435913+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Output":"ok \tgithub.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan\t(cached) [no tests to run]\n"}
{"Time":"2025-01-07T10:50:32.435919+01:00","Action":"pass","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/inferredspan","Elapsed":0}
{"Time":"2025-01-07T10:50:32.435982+01:00","Action":"start","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation"}
{"Time":"2025-01-07T10:50:32.436028+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"testing: warning: no tests to run\n"}
{"Time":"2025-01-07T10:50:32.436035+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"PASS\n"}
{"Time":"2025-01-07T10:50:32.436041+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Output":"ok \tgithub.com/DataDog/datadog-agent/pkg/serverless/trace/propagation\t(cached) [no tests to run]\n"}
{"Time":"2025-01-07T10:50:32.436046+01:00","Action":"pass","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace/propagation","Elapsed":0}
{"Time":"2025-01-07T10:50:33.460941+01:00","Action":"run","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast"}
{"Time":"2025-01-07T10:50:33.461002+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"=== RUN TestLoadConfigShouldBeFast\n"}
{"Time":"2025-01-07T10:50:33.464674+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"panic: toto\n"}
{"Time":"2025-01-07T10:50:33.464694+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\n"}
{"Time":"2025-01-07T10:50:33.464697+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"goroutine 213 [running]:\n"}
{"Time":"2025-01-07T10:50:33.464713+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"github.com/DataDog/datadog-agent/pkg/serverless/trace.TestLoadConfigShouldBeFast.func1()\n"}
{"Time":"2025-01-07T10:50:33.464726+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\t/Users/kevin.fairise/go/src/github.com/DataDog/datadog-agent/pkg/serverless/trace/trace_test.go:106 +0x2c\n"}
{"Time":"2025-01-07T10:50:33.464728+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"created by github.com/DataDog/datadog-agent/pkg/serverless/trace.TestLoadConfigShouldBeFast in goroutine 212\n"}
{"Time":"2025-01-07T10:50:33.46473+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Test":"TestLoadConfigShouldBeFast","Output":"\t/Users/kevin.fairise/go/src/github.com/DataDog/datadog-agent/pkg/serverless/trace/trace_test.go:105 +0x44\n"}
{"Time":"2025-01-07T10:50:33.465562+01:00","Action":"output","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Output":"FAIL\tgithub.com/DataDog/datadog-agent/pkg/serverless/trace\t1.029s\n"}
{"Time":"2025-01-07T10:50:33.465624+01:00","Action":"fail","Package":"github.com/DataDog/datadog-agent/pkg/serverless/trace","Elapsed":1.03}
39 changes: 39 additions & 0 deletions tasks/unit_tests/testwasher_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,45 @@ def test_should_not_be_considered_flaky(self):
{"github.com/DataDog/datadog-agent/test/new-e2e/tests/containers": {"TestEKSSuite"}},
)

def test_flaky_panicking_test(self):
test_washer = TestWasher(
test_output_json_file="test_output_failure_flaky_panic.json",
flakes_file_path="tasks/unit_tests/testdata/flakes_2.yaml",
)
module_path = "tasks/unit_tests/testdata"
failing_tests, marked_flaky_tests = test_washer.parse_test_results(module_path)
non_flaky_failing_tests = test_washer.get_non_flaky_failing_tests(
failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
)
self.assertEqual(non_flaky_failing_tests, {})

def test_non_flaky_panicking_test(self):
test_washer = TestWasher(
test_output_json_file="test_output_failure_panic.json",
flakes_file_path="tasks/unit_tests/testdata/flakes_2.yaml",
)
module_path = "tasks/unit_tests/testdata"
failing_tests, marked_flaky_tests = test_washer.parse_test_results(module_path)
non_flaky_failing_tests = test_washer.get_non_flaky_failing_tests(
failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
)
self.assertEqual(
non_flaky_failing_tests,
{'github.com/DataDog/datadog-agent/pkg/serverless/trace': {'TestLoadConfigShouldBeFast'}},
)

def test_flaky_panicking_flakesyaml_test(self):
test_washer = TestWasher(
test_output_json_file="test_output_failure_panic.json",
flakes_file_path="tasks/unit_tests/testdata/flakes_4.yaml",
)
module_path = "tasks/unit_tests/testdata"
failing_tests, marked_flaky_tests = test_washer.parse_test_results(module_path)
non_flaky_failing_tests = test_washer.get_non_flaky_failing_tests(
failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
)
self.assertEqual(non_flaky_failing_tests, {})


class TestMergeKnownFlakes(unittest.TestCase):
def test_with_shared_keys(self):
Expand Down

0 comments on commit f509427

Please sign in to comment.