From 51e98fa692f455bbb5e0b5fda6dc9f7751ebd8fa Mon Sep 17 00:00:00 2001 From: Papa Bakary Camara Date: Fri, 5 Jul 2024 16:53:24 +0200 Subject: [PATCH 1/5] Prepare run_in_docker script to use podman or docker --- run_in_docker.sh | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/run_in_docker.sh b/run_in_docker.sh index 436f0331..68188e00 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -2,7 +2,23 @@ # This script assumes that a Go service is already built, and the Python service can be interpreted without errors -# Function to get docker compose profile to add based on service name specified by the user +# Function to check if a command exists +command_exists() { + command -v "$1" &> /dev/null +} + +# Check if Podman is installed +if command_exists podman; then + CONTAINER_TOOL="podman" +# Check if Docker is installed +elif command_exists docker; then + CONTAINER_TOOL="docker" +else + echo "Neither Docker nor Podman is installed on this system." + exit 1 +fi + +# Function to get the correct profile to add based on service name specified by the user with_profile() { local target="$1" case "$target" in @@ -43,22 +59,22 @@ copy_go_executable() { local cid="$1" local path_to_service="$2" local executable_name="$3" - docker cp "$path_to_service/$executable_name" "$cid:$(docker exec "$cid" bash -c 'echo "$VIRTUAL_ENV_BIN"')" - docker exec -u root "$cid" /bin/bash -c "chmod +x \$VIRTUAL_ENV_BIN/$executable_name" + "$CONTAINER_TOOL" cp "$path_to_service/$executable_name" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$VIRTUAL_ENV_BIN"')" + "$CONTAINER_TOOL" exec -u root "$cid" /bin/bash -c "chmod +x \$VIRTUAL_ENV_BIN/$executable_name" } copy_python_project() { local cid="$1" local path_to_service="$2" - docker cp "$path_to_service" "$cid:/." + "$CONTAINER_TOOL" cp "$path_to_service" "$cid:/." # service will be pip-installed and executed, so user will need write and exec permissions - docker exec -u root "$cid" /bin/bash -c "chmod -R 777 /$(basename "$path_to_service")" + "$CONTAINER_TOOL" exec -u root "$cid" /bin/bash -c "chmod -R 777 /$(basename "$path_to_service")" } copy_openapi_spec() { local cid="$1" local path_to_service="$2" - docker cp "$path_to_service/openapi.json" "$cid:$(docker exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" } # Function to copy files based on the make target @@ -70,11 +86,11 @@ copy_files() { case "$target" in "aggregator-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator" - docker cp "$path_to_service/openapi.json" "$cid":"$(docker exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" ;; "aggregator-mock-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator-mock" - docker cp "$path_to_service" "$cid:$(docker exec "$cid" bash -c 'echo "$HOME"')/mock_server" + "$CONTAINER_TOOL" cp "$path_to_service" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')/mock_server" ;; "cleaner-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator-cleaner" @@ -98,7 +114,7 @@ copy_files() { "insights-content-service-tests") echo -e "\033[33mWARNING! Content service should include test-rules for these tests to run properly.\033[0m" echo -e "\033[33mPlease build using './build.sh --test-rules-only' or './build.sh --include-test-rules'\033[0m" - docker cp "$path_to_service" "$cid":"$(docker exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service" "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" ;; "insights-content-template-renderer-tests") copy_python_project "$cid" "$path_to_service" @@ -117,7 +133,7 @@ copy_files() { ;; "parquet-factory-tests") copy_go_executable "$cid" "$path_to_service" "parquet-factory" - docker cp "$path_to_service"/config.toml "$cid":"$(docker exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service"/config.toml "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" ;; *) echo "Unexpected target: $target. Does it exist in Makefile?" @@ -141,10 +157,10 @@ fi # Step 4: Launch containers # shellcheck disable=SC2046 -POSTGRES_DB_NAME="$db_name" docker compose $(with_profile "$1") $(with_no_mock "$3") up -d +POSTGRES_DB_NAME="$db_name" "$CONTAINER_TOOL-compose" $(with_profile "$1") $(with_no_mock "$3") up -d # Step 5: Find the container ID of the insights-behavioral-spec container -cid=$(docker ps | grep 'insights-behavioral-spec:latest' | cut -d ' ' -f 1) +cid=$("$CONTAINER_TOOL" ps | grep 'insights-behavioral-spec:latest' | cut -d ' ' -f 1) # Step 6: Copy the executable and needed dependencies or Python service into the container # TODO: Discuss including archives in compiled Go executables for testing @@ -153,4 +169,4 @@ copy_files "$cid" "$tests_target" "$path_to_service" # Step 9: Execute the specified make target -docker exec "$cid" /bin/bash -c "source \$VIRTUAL_ENV/bin/activate && env && $(with_mocked_dependencies "$3") make $tests_target" +"$CONTAINER_TOOL" exec "$cid" /bin/bash -c "source \$VIRTUAL_ENV/bin/activate && env && $(with_mocked_dependencies "$3") make $tests_target" From 7bbcc451b829b8cd88c80d81be3e19151571fe2f Mon Sep 17 00:00:00 2001 From: Papa Bakary Camara Date: Mon, 8 Jul 2024 10:57:49 +0200 Subject: [PATCH 2/5] Refactor to handle SC2016 --- run_in_docker.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/run_in_docker.sh b/run_in_docker.sh index 68188e00..4a87a4a3 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -18,6 +18,9 @@ else exit 1 fi +# shellcheck disable=SC2016 +REMOTE_PATH_FOR_SERVICE="$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + # Function to get the correct profile to add based on service name specified by the user with_profile() { local target="$1" @@ -59,6 +62,7 @@ copy_go_executable() { local cid="$1" local path_to_service="$2" local executable_name="$3" + # shellcheck disable=SC2016 "$CONTAINER_TOOL" cp "$path_to_service/$executable_name" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$VIRTUAL_ENV_BIN"')" "$CONTAINER_TOOL" exec -u root "$cid" /bin/bash -c "chmod +x \$VIRTUAL_ENV_BIN/$executable_name" } @@ -74,7 +78,7 @@ copy_python_project() { copy_openapi_spec() { local cid="$1" local path_to_service="$2" - "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$REMOTE_PATH_FOR_SERVICE" } # Function to copy files based on the make target @@ -86,11 +90,11 @@ copy_files() { case "$target" in "aggregator-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator" - "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service/openapi.json" "$REMOTE_PATH_FOR_SERVICE" ;; "aggregator-mock-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator-mock" - "$CONTAINER_TOOL" cp "$path_to_service" "$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')/mock_server" + "$CONTAINER_TOOL" cp "$path_to_service" "$REMOTE_PATH_FOR_SERVICE/mock_server" ;; "cleaner-tests") copy_go_executable "$cid" "$path_to_service" "insights-results-aggregator-cleaner" @@ -114,7 +118,7 @@ copy_files() { "insights-content-service-tests") echo -e "\033[33mWARNING! Content service should include test-rules for these tests to run properly.\033[0m" echo -e "\033[33mPlease build using './build.sh --test-rules-only' or './build.sh --include-test-rules'\033[0m" - "$CONTAINER_TOOL" cp "$path_to_service" "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service" "$REMOTE_PATH_FOR_SERVICE" ;; "insights-content-template-renderer-tests") copy_python_project "$cid" "$path_to_service" @@ -133,7 +137,7 @@ copy_files() { ;; "parquet-factory-tests") copy_go_executable "$cid" "$path_to_service" "parquet-factory" - "$CONTAINER_TOOL" cp "$path_to_service"/config.toml "$cid":"$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + "$CONTAINER_TOOL" cp "$path_to_service"/config.toml "$REMOTE_PATH_FOR_SERVICE" ;; *) echo "Unexpected target: $target. Does it exist in Makefile?" @@ -157,7 +161,7 @@ fi # Step 4: Launch containers # shellcheck disable=SC2046 -POSTGRES_DB_NAME="$db_name" "$CONTAINER_TOOL-compose" $(with_profile "$1") $(with_no_mock "$3") up -d +POSTGRES_DB_NAME="$db_name" "$CONTAINER_TOOL compose" $(with_profile "$1") $(with_no_mock "$3") up -d # Step 5: Find the container ID of the insights-behavioral-spec container cid=$("$CONTAINER_TOOL" ps | grep 'insights-behavioral-spec:latest' | cut -d ' ' -f 1) @@ -166,7 +170,5 @@ cid=$("$CONTAINER_TOOL" ps | grep 'insights-behavioral-spec:latest' | cut -d ' ' # TODO: Discuss including archives in compiled Go executables for testing copy_files "$cid" "$tests_target" "$path_to_service" -# Step 9: Execute the specified make target - - +# Step 7: Execute the specified make target "$CONTAINER_TOOL" exec "$cid" /bin/bash -c "source \$VIRTUAL_ENV/bin/activate && env && $(with_mocked_dependencies "$3") make $tests_target" From d40cf4af5ea90576a5f32dd20f405af1f48fed2a Mon Sep 17 00:00:00 2001 From: Papa Bakary Camara Date: Fri, 9 Aug 2024 10:06:36 +0200 Subject: [PATCH 3/5] Use proper composer --- run_in_docker.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run_in_docker.sh b/run_in_docker.sh index 4a87a4a3..af819dc2 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -10,17 +10,16 @@ command_exists() { # Check if Podman is installed if command_exists podman; then CONTAINER_TOOL="podman" + COMPOSER="podman-compose" # Check if Docker is installed elif command_exists docker; then CONTAINER_TOOL="docker" + COMPOSER="docker compose" else echo "Neither Docker nor Podman is installed on this system." exit 1 fi -# shellcheck disable=SC2016 -REMOTE_PATH_FOR_SERVICE="$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" - # Function to get the correct profile to add based on service name specified by the user with_profile() { local target="$1" @@ -161,11 +160,14 @@ fi # Step 4: Launch containers # shellcheck disable=SC2046 -POSTGRES_DB_NAME="$db_name" "$CONTAINER_TOOL compose" $(with_profile "$1") $(with_no_mock "$3") up -d +POSTGRES_DB_NAME="$db_name" "$COMPOSER" $(with_profile "$1") $(with_no_mock "$3") up -d # Step 5: Find the container ID of the insights-behavioral-spec container cid=$("$CONTAINER_TOOL" ps | grep 'insights-behavioral-spec:latest' | cut -d ' ' -f 1) +# shellcheck disable=SC2016 +REMOTE_PATH_FOR_SERVICE="$cid:$("$CONTAINER_TOOL" exec "$cid" bash -c 'echo "$HOME"')" + # Step 6: Copy the executable and needed dependencies or Python service into the container # TODO: Discuss including archives in compiled Go executables for testing copy_files "$cid" "$tests_target" "$path_to_service" From 6637fd90e9a5d27f45373a6b6235d3a25108c33d Mon Sep 17 00:00:00 2001 From: Papa Bakary Camara Date: Fri, 9 Aug 2024 10:17:15 +0200 Subject: [PATCH 4/5] Add more health checks in docker-compose --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4ae94481..2dba9e1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,12 @@ services: environment: - KAFKA_ADVERTISED_HOST_NAME=kafka - KAFKA_CREATE_TOPICS="platform.notifications.ingress:1:1" + healthcheck: + test: ["CMD", "bash", "-c", "echo > /dev/tcp/localhost/9092"] + interval: 5s + timeout: 2s + retries: 5 + start_period: 5s minio: profiles: From 152c429a0fff7a571d212c7a3744c8c25ce9e62a Mon Sep 17 00:00:00 2001 From: Papa Bakary Camara Date: Fri, 9 Aug 2024 10:47:36 +0200 Subject: [PATCH 5/5] Switch order of checks --- run_in_docker.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/run_in_docker.sh b/run_in_docker.sh index af819dc2..47a78314 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -7,16 +7,17 @@ command_exists() { command -v "$1" &> /dev/null } -# Check if Podman is installed -if command_exists podman; then - CONTAINER_TOOL="podman" - COMPOSER="podman-compose" + # Check if Docker is installed -elif command_exists docker; then +if command_exists docker; then CONTAINER_TOOL="docker" COMPOSER="docker compose" +# Check if Podman is installed +elif command_exists podman; then + CONTAINER_TOOL="podman" + COMPOSER="podman-compose" else - echo "Neither Docker nor Podman is installed on this system." + echo "Neither Docker nor Podman are installed on this system." exit 1 fi