feat: GCI-11 - Container support for Generic CI (#908)

* feat: GCI-11: Workflow container experiment

* feat: GCI-11 refactor docker manager

* feat: GCI-11 use new docker manager for services

* feat: GCI-11 do not override command for services

* fix: GCI-11 make services great again

* fix: GCI-11 remove debug + fix containername

* debug: GCI-11 add logging for service containers

* fix: GCI-11 fix workdir

* feat: GCI-11: Ensuring network is created when needed

* GCI-11: fixed possible nil pointer panic

* GCI-11: Handle not existing docker container before step exec

* feat: GCI-11 introduce docker client (testing)

* feat: GCI-12 support options and healthcheck

* feat: GCI-11: ensureNetwork() uses native docker client lib

* feat: GCI-11: Added step logging about container

* feat: GCI-12 Start all service container before healthcheck

* feat: GCI-12 do not use errors as that requires newer go

* feat: GCI-12 return any created containers to make sure it gets cleaned up

* feat: GCI-12 start workflow container as root

* feat: GCI-11 refactor healthcheck with ID + pretty logging

* fix: GCI-12 fix container log on error

* fix: GCI-12 refactor start container

* fix: GCI-12 run all services even when some of the mfails

* feat: GCI-11: Added signal handling and cleanup

* retrigger checks

* chore: GCI-11 remove todo comments that have been resolved

* chore: GCI-11 remove debug log from env passthrough

* feat: GCI-11 add separate docker pull command with retry

* debug: GCI-11 debug logging

* feat: GCI-11 use cli for pulling instead of sdk

* feat: GCI-11 refactor run container

* feat: GCI-11 enable resolving env vars in container config

* feat: GCI-11 enable logging with redacted variables

* feat: GCI-11 container manager has a custom logger, which takes care of redacting secret values

* feat: GCI-11 added docker related tests with a local test run environment setup

* fix: GCI-11 docker-compose and dockerfile are reverted back to the original state

* fix: GCI-11 Making linter happy

* fix: GCI-11 separating docker tests to run only on linux

* fix: GCI-11 fix relative paths in tests

* fix: GCI-11 fix relative paths in tests

* fix: GCI-11 removed debug logs

* feat: GCI-11 bump version

* feat: GCI-11 Nitpick based on PR feedback

* feat: GCI-11 Nitpicks based on PR feedback, part 2

* feat: GCI-11 Fixed container integration test

* feat: GCI-11 - Added 'CI' to container envs whitelist

* feat: GCI-11 - Comments into makefile

* feat: GCI-11 - rm volumes as well on cleanup

---------

Co-authored-by: Olivér Falvai <[email protected]>
Co-authored-by: Balázs Rostás <[email protected]>

.gitignore

@@ -1,5 +1,6 @@
 _temp/
 _tmp/
+_local/
 .bitrise
 .bitrise.secrets.yml
 _bin

Makefile

@@ -0,0 +1,14 @@
+# this is intended to be used only for local testing of container related integration tests
+# everything else CI related is in the bitrise.yml
+
+DOCKER_COMPOSE_FILE=_tests/integration/local_docker_test_environment/docker-compose.yml
+SRC_DIR_IN_GOPATH=/bitrise/src
+
+docker-test: setup-test-environment
+	docker exec -it bitrise-main-container bash -c "export INTEGRATION_TEST_BINARY_PATH=\$$PWD/bitrise-cli; go test ./_tests/integration -tags linux_only"
+
+setup-test-environment: build-main-container
+	docker exec -it bitrise-main-container bash -c "go build -o bitrise-cli"
+
+build-main-container:
+	docker-compose -f $(DOCKER_COMPOSE_FILE) up --build -d

_tests/integration/agent_config_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (

_tests/integration/async_step_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (

_tests/integration/docker_create_bitrise.yml

@@ -0,0 +1,47 @@
+format_version: 1.3.0
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+workflows:
+  docker-create-fails-invalid-port:
+    title: Expected to fail on docker create, invalid port provided
+    container:
+      image: frolvlad/alpine-bash:latest
+      ports:
+      - 22:22
+    steps:
+    - script:
+        title: Should not run due to prev error
+        inputs:
+        - content: exit 0
+  docker-create-succeeds-valid-port:
+    title: Expected to pass on docker create, valid port provided
+    container:
+      image: frolvlad/alpine-bash:latest
+      ports:
+      - 12341:12341
+    steps:
+    - script:
+        title: Should succeed
+        inputs:
+        - content: exit 0
+  docker-create-succeeds-with-false-unhealthy-container:
+    title: Expected to log error on docker create
+    description: Expected to log error on docker create, because healthchecks are wrong, however execution should continue
+    container:
+      image: frolvlad/alpine-bash:latest
+      options: --health-cmd "redis-cli ping" --health-interval 1s --health-timeout 3s --health-retries 2
+    steps:
+    - script:
+        title: Should succceed
+        inputs:
+        - content: exit 0
+  docker-create-fails-invalid-option:
+    title: Expected to log error on docker create
+    description: Expected to log error on docker create, because healthcheck are wrong, however execution should continue
+    container:
+      image: frolvlad/alpine-bash:latest
+      options: --invalid-option "fail now!"
+    steps:
+    - script:
+        title: Should fail
+        inputs:
+        - content: exit 0

_tests/integration/docker_login_secrets.yml

@@ -0,0 +1,3 @@
+envs:
+- BITRISE_SECRET_FILTERING: true
+- DOCKER_PW: test

_tests/integration/docker_pull_bitrise.yml

@@ -0,0 +1,71 @@
+format_version: 1.3.0
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+workflows:
+  docker-pull-success:
+    title: Expected to pass docker pull
+    container:
+      image: frolvlad/alpine-bash:latest
+    steps:
+    - script:
+        title: Should pass
+        inputs:
+        - content: exit 0
+  docker-pull-fails-404:
+    title: Expected to fail docker pull
+    container:
+      image: localhost.hu/noimage:3cb48a46a66e
+    steps:
+    - script:
+        title: Should fail
+        inputs:
+        - content: exit 0
+  docker-login-fail:
+    title: Expected to fail on docker login
+    container:
+      image: us-central1-docker.pkg.dev/ip-kubernetes-dev/sandbox/ruby:zstd
+      credentials:
+        username: _json_key_base64
+        password: bad pw
+    steps:
+    - script:
+        title: Should fail
+        inputs:
+        - content: exit 0
+  docker-login-success:
+    before_run:
+    - _start_mock_registry
+    after_run:
+    - _cleanup_mock_registry
+    title: Expected to pass docker login
+    container:
+      image: localhost:5001/frolvlad/alpine-bash:latest
+      credentials:
+        username: test
+        password: $DOCKER_PW
+    steps:
+    - script:
+        title: Should pass
+        inputs:
+        - content: exit 0
+  _start_mock_registry:
+    steps:
+    - script:
+        title: setup mock registry
+        inputs:
+        - content: |-
+            docker pull --platform linux/amd64 registry:latest
+            docker run -d -p 5001:5000 --restart always --name registry registry
+            docker pull --platform linux/amd64 frolvlad/alpine-bash:latest
+            docker login localhost:5001 -u test -p test
+            docker tag frolvlad/alpine-bash:latest localhost:5001/frolvlad/alpine-bash:latest
+            docker push localhost:5001/frolvlad/alpine-bash:latest
+            docker logout localhost:5001
+  _cleanup_mock_registry:
+    steps:
+    - script:
+        is_always_run: true
+        title: cleanup mock registry
+        inputs:
+        - content: |-
+            docker stop registry
+            docker rm registry

_tests/integration/docker_service_bitrise.yml

@@ -0,0 +1,41 @@
+format_version: 1.3.0
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+workflows:
+  docker-service-start-fails:
+    before_run:
+    - _build-failing-image
+    services:
+      failing-service:
+        image: test-failing-image
+    steps:
+    - script:
+        title: Should succeed, but services related errors are logged
+        inputs:
+        - content: exit 0
+  docker-service-start-succeeds-after-retries:
+    before_run:
+    - _build-slow-starting-image
+    services:
+      slow-bootin-service:
+        image: test-slow-booting-image
+        options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 16
+    steps:
+    - script:
+        title: Should succeed, but services related errors are logged
+        inputs:
+        - content: exit 0
+
+  _build-failing-image:
+    steps:
+    - script:
+        title: Build failing image
+        inputs:
+        - content: |-
+              docker build -t test-failing-image -f ${SRC_DIR_IN_GOPATH}/_tests/integration/docker_test.Dockerfile.failing-container .
+  _build-slow-starting-image:
+    steps:
+    - script:
+        title: Build slow starting image
+        inputs:
+        - content: |-
+              docker build -t test-slow-booting-image -f ${SRC_DIR_IN_GOPATH}/_tests/integration/docker_test.Dockerfile.slow-booting-container .

_tests/integration/docker_start_fails_bitrise.yml

@@ -0,0 +1,21 @@
+format_version: 1.3.0
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+workflows:
+  docker-start-fails:
+    before_run:
+    - _build-failing-image
+    title: Expected to fail on docker start, failing image is used
+    container:
+      image: test-failing-image:latest
+    steps:
+    - script:
+        title: Should not run due to prev error
+        inputs:
+        - content: exit 0
+  _build-failing-image:
+    steps:
+    - script:
+        title: Build failing image
+        inputs:
+        - content: |-
+            docker build -t test-failing-image -f ${SRC_DIR_IN_GOPATH}/_tests/integration/docker_test.Dockerfile.failing-container .

_tests/integration/docker_test.Dockerfile.failing-container

@@ -0,0 +1,2 @@
+FROM frolvlad/alpine-bash:latest
+ENTRYPOINT ["nonexistent-command"]

_tests/integration/docker_test.Dockerfile.slow-booting-container

@@ -0,0 +1,2 @@
+FROM frolvlad/alpine-bash:latest
+CMD ["bash", "-c", "sleep 3; touch /ready; echo 'Im healthy now'; sleep infinity"]

_tests/integration/docker_test.go

@@ -0,0 +1,136 @@
+//go:build linux_only
+// +build linux_only
+
+package integration
+
+import (
+	"testing"
+
+	"github.com/bitrise-io/go-utils/command"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Docker(t *testing.T) {
+	testCases := map[string]struct {
+		configPath    string
+		inventoryPath string
+		workflowName  string
+		requireErr    bool
+		requireLogs   []string
+	}{
+		"docker pull succeeds with existing image": {
+			configPath:   "docker_pull_bitrise.yml",
+			workflowName: "docker-pull-success",
+			requireErr:   false,
+			requireLogs:  []string{"Step is running in container:"},
+		},
+		"docker pull fails with non-existing image": {
+			configPath:   "docker_pull_bitrise.yml",
+			workflowName: "docker-pull-fails-404",
+			requireErr:   true,
+			requireLogs: []string{
+				"Error during image pull",
+				"Failed to pull image, retrying",
+			},
+		},
+		"docker login fails when incorrect credentials are provided": {
+			configPath:   "docker_pull_bitrise.yml",
+			workflowName: "docker-login-fail",
+			requireErr:   true,
+			requireLogs: []string{
+				"workflow has docker credentials provided, but the authentication failed",
+			},
+		},
+		"docker login succeeds when correct credentials are provided": {
+			configPath:    "docker_pull_bitrise.yml",
+			inventoryPath: "docker_login_secrets.yml",
+			workflowName:  "docker-login-success",
+			requireErr:    false,
+			requireLogs: []string{
+				"Logging into docker registry:",
+				"Step is running in container:",
+				"--password [REDACTED]",
+			},
+		},
+		"docker create fails when already-used port is provided": {
+			configPath:   "docker_create_bitrise.yml",
+			workflowName: "docker-create-fails-invalid-port",
+			requireErr:   true,
+			requireLogs: []string{
+				"failed to start containers:",
+				"bind: address already in use",
+			},
+		},
+		"docker create succeeds when valid port is provided": {
+			configPath:   "docker_create_bitrise.yml",
+			workflowName: "docker-create-succeeds-valid-port",
+			requireErr:   false,
+			requireLogs: []string{
+				"Step is running in container:",
+			},
+		},
+		"docker create succeeds if false negative health check result is present": {
+			configPath:   "docker_create_bitrise.yml",
+			workflowName: "docker-create-succeeds-with-false-unhealthy-container",
+			requireErr:   false,
+			requireLogs: []string{
+				"Container (bitrise-workflow-docker-create-succeeds-with-false-unhealthy-container) is unhealthy...",
+				"Step is running in container: frolvlad/alpine-bash:latest",
+			},
+		},
+		"docker create fails when invalid option is provided": {
+			configPath:   "docker_create_bitrise.yml",
+			workflowName: "docker-create-fails-invalid-option",
+			requireErr:   true,
+			requireLogs: []string{
+				"unknown flag: --invalid-option",
+				"Could not start the specified docker image for workflow:",
+			},
+		},
+		"docker start fails with container throwing error": {
+			configPath:   "docker_start_fails_bitrise.yml",
+			workflowName: "docker-start-fails",
+			requireErr:   true,
+			requireLogs: []string{
+				"nonexistent-command",
+			},
+		},
+		"docker service start fails, build succeeds, service error is logged": {
+			configPath:   "docker_service_bitrise.yml",
+			workflowName: "docker-service-start-fails",
+			requireErr:   false,
+			requireLogs: []string{
+				"Some services failed to start properly",
+				"start docker container (failing-service): exit status 1",
+				"nonexistent-command",
+			},
+		},
+		"docker start services succeeds after retries": {
+			configPath:   "docker_service_bitrise.yml",
+			workflowName: "docker-service-start-succeeds-after-retries",
+			requireErr:   false,
+			requireLogs: []string{
+				"Waiting for container (slow-bootin-service) to be healthy",
+			},
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			cmd := command.New(binPath(), "run", testCase.workflowName, "--config", testCase.configPath)
+			if testCase.inventoryPath != "" {
+				cmd.GetCmd().Args = append(cmd.GetCmd().Args, "--inventory", testCase.inventoryPath)
+			}
+
+			out, err := cmd.RunAndReturnTrimmedCombinedOutput()
+			if testCase.requireErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			for _, log := range testCase.requireLogs {
+				require.Contains(t, out, log)
+			}
+		})
+	}
+}

_tests/integration/envstore_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (

_tests/integration/exit_code_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (

_tests/integration/global_flag_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (

_tests/integration/gomodmigrate_test.go

@@ -1,3 +1,6 @@
+//go:build linux_and_mac
+// +build linux_and_mac
+
 package integration
 
 import (