reporting abstraction

Signed-off-by: Marcin Owsiany <[email protected]>

pkg/report/report.go

@@ -114,6 +114,20 @@ type Testsuites struct {
 	lock    sync.Mutex
 }
 
+// StepReporter is an interface for reporting status of a test step.
+type StepReporter interface {
+	Failure(message string, errors ...error)
+	AddAssertions(i int)
+}
+
+// TestReporter is an interface for reporting status of a test.
+// For each step, call Step and use the returned step reporter.
+// Make sure to call Done when a test ends (preferably using defer).
+type TestReporter interface {
+	Step(stepName string) StepReporter
+	Done()
+}
+
 // NewSuiteCollection returns the address of a newly created TestSuites
 func NewSuiteCollection(name string) *Testsuites {
 	start := time.Now()
@@ -209,6 +223,54 @@ func (ts *Testsuite) summarize() time.Time {
 	return end
 }
 
+func (ts *Testsuite) NewTest(name string) TestReporter {
+	subSuite := ts.NewSubSuite(name)
+	return &testReporter{suite: subSuite}
+}
+
+type stepReport struct {
+	name       string
+	failed     bool
+	failureMsg string
+	errors     []error
+	assertions int
+}
+
+func (s *stepReport) Failure(message string, errors ...error) {
+	s.failed = true
+	s.failureMsg = message
+	s.errors = append(s.errors, errors...)
+}
+
+func (s *stepReport) AddAssertions(i int) {
+	s.assertions += i
+}
+
+type testReporter struct {
+	suite       *Testsuite
+	stepReports []*stepReport
+}
+
+func (r *testReporter) Step(stepName string) StepReporter {
+	step := &stepReport{name: stepName}
+	r.stepReports = append(r.stepReports, step)
+	return step
+}
+
+func (r *testReporter) Done() {
+	for _, report := range r.stepReports {
+		testCase := NewCase(report.name)
+		if report.failed {
+			testCase.Failure = NewFailure(report.failureMsg, report.errors)
+		}
+		testCase.Assertions += report.assertions
+		r.suite.AddTestcase(testCase)
+	}
+}
+
+var _ TestReporter = (*testReporter)(nil)
+var _ StepReporter = (*stepReport)(nil)
+
 // AddTestSuite is a convenience method to add a testsuite to the collection in testsuites
 func (ts *Testsuites) AddTestSuite(testsuite *Testsuite) {
 	// testsuite is added prior to stat availability, stat management in the close of the testsuites

pkg/test/case.go

@@ -316,19 +316,18 @@ func shortString(obj *corev1.ObjectReference) string {
 }
 
 // Run runs a test case including all of its steps.
-func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
-	setupReport := report.NewCase("setup")
+func (t *Case) Run(test *testing.T, rep report.TestReporter) {
+	defer rep.Done()
+	setupReport := rep.Step("setup")
 	ns, err := t.determineNamespace()
 	if err != nil {
-		setupReport.Failure = report.NewFailure(err.Error(), nil)
-		ts.AddTestcase(setupReport)
+		setupReport.Failure(err.Error())
 		test.Fatal(err)
 	}
 
 	cl, err := t.Client(false)
 	if err != nil {
-		setupReport.Failure = report.NewFailure(err.Error(), nil)
-		ts.AddTestcase(setupReport)
+		setupReport.Failure(err.Error())
 		test.Fatal(err)
 	}
 
@@ -341,8 +340,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 
 		cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false)
 		if err != nil {
-			setupReport.Failure = report.NewFailure(err.Error(), nil)
-			ts.AddTestcase(setupReport)
+			setupReport.Failure(err.Error())
 			test.Fatal(err)
 		}
 
@@ -353,15 +351,13 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 		if err = t.CreateNamespace(test, c, ns); k8serrors.IsAlreadyExists(err) {
 			t.Logger.Logf("namespace %q already exists, using kubeconfig %q", ns.Name, kc)
 		} else if err != nil {
-			setupReport.Failure = report.NewFailure("failed to create test namespace", []error{err})
-			ts.AddTestcase(setupReport)
+			setupReport.Failure("failed to create test namespace", err)
 			test.Fatal(err)
 		}
 	}
-	ts.AddTestcase(setupReport)
 
 	for _, testStep := range t.Steps {
-		tc := report.NewCase("step " + testStep.String())
+		stepReport := rep.Step("step " + testStep.String())
 		testStep.Client = t.Client
 		if testStep.Kubeconfig != "" {
 			testStep.Client = newClient(testStep.Kubeconfig, testStep.Context)
@@ -371,8 +367,8 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 			testStep.DiscoveryClient = newDiscoveryClient(testStep.Kubeconfig, testStep.Context)
 		}
 		testStep.Logger = t.Logger.WithPrefix(testStep.String())
-		tc.Assertions += len(testStep.Asserts)
-		tc.Assertions += len(testStep.Errors)
+		stepReport.AddAssertions(len(testStep.Asserts))
+		stepReport.AddAssertions(len(testStep.Errors))
 
 		errs := []error{}
 
@@ -395,15 +391,12 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 
 		if len(errs) > 0 {
 			caseErr := fmt.Errorf("failed in step %s", testStep.String())
-			tc.Failure = report.NewFailure(caseErr.Error(), errs)
+			stepReport.Failure(caseErr.Error(), errs...)
 
 			test.Error(caseErr)
 			for _, err := range errs {
 				test.Error(err)
 			}
-		}
-		ts.AddTestcase(tc)
-		if len(errs) > 0 {
 			break
 		}
 	}

pkg/test/case_integration_test.go

@@ -111,5 +111,14 @@ func TestMultiClusterCase(t *testing.T) {
 		},
 	}
 
-	c.Run(t, &report.Testsuite{})
+	c.Run(t, &noOpReporter{})
 }
+
+type noOpReporter struct{}
+
+func (r *noOpReporter) Done() {}
+func (r *noOpReporter) Step(string) report.StepReporter {
+	return r
+}
+func (r *noOpReporter) AddAssertions(int)        {}
+func (r *noOpReporter) Failure(string, ...error) {}

pkg/test/harness.go

@@ -397,8 +397,8 @@ func (h *Harness) RunTests() {
 						t.Fatal(err)
 					}
 
-					testReport := suiteReport.NewSubSuite(test.Name)
-					test.Run(t, testReport)
+					testReporter := suiteReport.NewTest(test.Name)
+					test.Run(t, testReporter)
 				})
 			}
 		}