Print container logs when integration test fails (#1127)

We see (relatively rarely) integration tests being flaky. In those cases
of flakiness the current logging is not sufficient to really understand
the cause of the problem.

This PR aims to improve logging by printing logs of all containers when
a test fails.

`printContainerLogs` is added to `Containers.Cleanup()` code to print
the logs of containers.

Additionally, the `Cleanup()` function is now **always** called, even if
the containers failed to start or tests failed. To make sure it can
print some logs:
- the code now has less panics, which would have prevented `Cleanup()`
from running
- in `setupClickHouse()` and others we return the container object in
case container failed to start instead of `nil` (but still with error)
to be able to recover some logs
- if one container fails to start, `Containers` is now returned
partially filled (but still with error), so that we can print some logs
instead of giving up entirely

---------

Signed-off-by: Piotr Grabowski <[email protected]>
Co-authored-by: Jacek Migdal <[email protected]>

ci/it/integration_test.go

@@ -11,13 +11,13 @@ import (
 
 func runIntegrationTest(t *testing.T, testCase testcases.TestCase) {
 	ctx := context.Background()
+	defer testCase.Cleanup(ctx, t)
 	if err := testCase.SetupContainers(ctx); err != nil {
 		t.Fatalf("Failed to setup containers: %s", err)
 	}
 	if err := testCase.RunTests(ctx, t); err != nil {
 		t.Fatalf("Failed to run tests: %s", err)
 	}
-	testCase.Cleanup(ctx)
 }
 
 func TestTransparentProxy(t *testing.T) {

ci/it/testcases/base.go

@@ -20,7 +20,7 @@ import (
 type TestCase interface {
 	SetupContainers(ctx context.Context) error
 	RunTests(ctx context.Context, t *testing.T) error
-	Cleanup(ctx context.Context)
+	Cleanup(ctx context.Context, t *testing.T)
 }
 
 type IntegrationTestcaseBase struct {
@@ -36,8 +36,10 @@ func (tc *IntegrationTestcaseBase) RunTests(ctx context.Context, t *testing.T) e
 	return nil
 }
 
-func (tc *IntegrationTestcaseBase) Cleanup(ctx context.Context) {
-	tc.Containers.Cleanup(ctx)
+func (tc *IntegrationTestcaseBase) Cleanup(ctx context.Context, t *testing.T) {
+	if tc.Containers != nil {
+		tc.Containers.Cleanup(ctx, t)
+	}
 }
 
 func (tc *IntegrationTestcaseBase) getQuesmaEndpoint() string {

ci/it/testcases/test_ab.go

@@ -34,11 +34,8 @@ func NewABTestcase() *ABTestcase {
 
 func (a *ABTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *ABTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_dual_write_and_common_table.go

@@ -26,11 +26,8 @@ func NewDualWriteAndCommonTableTestcase() *DualWriteAndCommonTableTestcase {
 
 func (a *DualWriteAndCommonTableTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *DualWriteAndCommonTableTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_ingest.go

@@ -28,11 +28,8 @@ func NewIngestTestcase() *IngestTestcase {
 
 func (a *IngestTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *IngestTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_reading_clickhouse_tables.go

@@ -26,11 +26,8 @@ func NewReadingClickHouseTablesIntegrationTestcase() *ReadingClickHouseTablesInt
 
 func (a *ReadingClickHouseTablesIntegrationTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *ReadingClickHouseTablesIntegrationTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_transparent_proxy.go

@@ -25,11 +25,8 @@ func NewTransparentProxyIntegrationTestcase() *TransparentProxyIntegrationTestca
 
 func (a *TransparentProxyIntegrationTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupContainersForTransparentProxy(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *TransparentProxyIntegrationTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_two_pipelines.go

@@ -25,11 +25,8 @@ func NewQueryAndIngestPipelineTestcase() *QueryAndIngestPipelineTestcase {
 
 func (a *QueryAndIngestPipelineTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *QueryAndIngestPipelineTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_wildcard_clickhouse.go

@@ -25,11 +25,8 @@ func NewWildcardClickhouseTestcase() *WildcardClickhouseTestcase {
 
 func (a *WildcardClickhouseTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *WildcardClickhouseTestcase) RunTests(ctx context.Context, t *testing.T) error {

ci/it/testcases/test_wildcard_disabled.go

@@ -25,11 +25,8 @@ func NewWildcardDisabledTestcase() *WildcardDisabledTestcase {
 
 func (a *WildcardDisabledTestcase) SetupContainers(ctx context.Context) error {
 	containers, err := setupAllContainersWithCh(ctx, a.ConfigTemplate)
-	if err != nil {
-		return err
-	}
 	a.Containers = containers
-	return nil
+	return err
 }
 
 func (a *WildcardDisabledTestcase) RunTests(ctx context.Context, t *testing.T) error {