From cb7b0cfbe47348adbeda53ce142d14a38c76ee4b Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Fri, 20 Dec 2024 11:56:27 +0100 Subject: [PATCH] Fix one of the causes of IT flakiness: port conflicts between IPv4 and IPv6 (#1132) The integration tests rare flakiness (~1/16 chance) seems to have multiple causes. This PR fixes one of them - flakiness caused by port conflicts between IPv4 and IPv6. In one of the flaky runs, `QueryAndIngestPipelineTestcase.testWildcardGoesToElastic` panicked while trying to parse results returned from Quesma (Quesma seemingly returned something really incorrect!): ```go resp, bodyBytes := a.RequestToQuesma(ctx, t, "POST", "/unmentioned_index/_search", []byte(`{"query": {"match_all": {}}}`)) var jsonResponse map[string]interface{} if err := json.Unmarshal(bodyBytes, &jsonResponse); err != nil { t.Fatalf("Failed to unmarshal response body: %s", err) } hits, _ := jsonResponse["hits"].(map[string]interface{}) // PANIC HERE ``` Thankfully I collected a tcpdump from this run, which showed something strange: Request to Quesma, but Kibana responds Dumping all containers info from Docker: ``` Quesma container: "Ports": { "8080/tcp": [{ "HostIp": "0.0.0.0", "HostPort": "32790" }, { "HostIp": "::", "HostPort": "32789" }], }, Kibana container: "Ports": { "5601/tcp": [{ "HostIp": "0.0.0.0", "HostPort": "32791" }, { "HostIp": "::", "HostPort": "32790" }] }, ``` Quesma container is bound to `0.0.0.0:32790` (IPv4) and Kibana container is bound to `::0:32790` (IPv6). When we make requests to Quesma, we connect to `localhost:32790` - depending on the codepath sometimes it connects to the correct IPv4 Quesma endpoint, but sometimes it uses IPv6 and erroneously connects to Kibana instead. This is a bug in testcontainers-go: https://github.com/testcontainers/testcontainers-go/issues/551 A suggested solution on the linked issue is to bind ports to host only by IPv4 - this PR applies such workaround. Note that there are some other sources of flakiness in Quesma's ITs still left to debug and fix - this only fixes one of them. --- ci/it/testcases/utils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/it/testcases/utils.go b/ci/it/testcases/utils.go index 8578903d0..89b6143bd 100644 --- a/ci/it/testcases/utils.go +++ b/ci/it/testcases/utils.go @@ -52,7 +52,7 @@ func (c *Containers) Cleanup(ctx context.Context) { func setupElasticsearch(ctx context.Context) (testcontainers.Container, error) { req := testcontainers.ContainerRequest{ Image: "docker.elastic.co/elasticsearch/elasticsearch:8.11.1", - ExposedPorts: []string{"9200/tcp", "9300/tcp"}, + ExposedPorts: []string{"0.0.0.0::9200/tcp", "0.0.0.0::9300/tcp"}, // Do i ned Env: map[string]string{ "discovery.type": "single-node", @@ -99,7 +99,7 @@ func setupQuesma(ctx context.Context, quesmaConfig string) (testcontainers.Conta } quesmaReq := testcontainers.ContainerRequest{ Image: "quesma/quesma:nightly", - ExposedPorts: []string{"9999/tcp", "8080/tcp"}, + ExposedPorts: []string{"0.0.0.0::9999/tcp", "0.0.0.0::8080/tcp"}, Env: map[string]string{ "QUESMA_CONFIG_FILE": "/configuration/conf.yaml", }, @@ -138,7 +138,7 @@ func setupKibana(ctx context.Context, quesmaContainer testcontainers.Container) req := testcontainers.ContainerRequest{ Image: "docker.elastic.co/kibana/kibana:8.11.1", - ExposedPorts: []string{"5601/tcp"}, + ExposedPorts: []string{"0.0.0.0::5601/tcp"}, Env: map[string]string{ "ELASTICSEARCH_HOSTS": fmt.Sprintf("[\"%s\"]", fmt.Sprintf("http://%s:%s", GetInternalDockerHost(), port.Port())), "XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY": "QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA", @@ -165,7 +165,7 @@ func setupKibana(ctx context.Context, quesmaContainer testcontainers.Container) func setupClickHouse(ctx context.Context) (testcontainers.Container, error) { req := testcontainers.ContainerRequest{ Image: "clickhouse/clickhouse-server:24.5.3.5-alpine", - ExposedPorts: []string{"8123/tcp", "9000/tcp"}, + ExposedPorts: []string{"0.0.0.0::8123/tcp", "0.0.0.0::9000/tcp"}, HostConfigModifier: func(hc *container.HostConfig) { hc.ExtraHosts = []string{"localhost-for-github-ci:host-gateway"} },