Add CI_SERVICE_alias environment variables to make referencing the FQ…

…DN of services fun! (#27) * Add config.sh to setup custom environment variables `config.sh` prints JSON to STDOUT following the expected form defined in https://docs.gitlab.com/runner/executors/custom.html#config This change also moves `base.sh` warnings to STDERR. * Add CI_SERVICE_alias environment variables Much ado about little here. Some restructuring was needed in order to populate a CI_SERVICE_<alias> variables as services are started, then inject them into a temporary manifest. On the way I found that the old TMPVARFILE was not working correctly. I 🪿 love 🪿 bash🪿!!! * Add README section for use of the runner
GSA-TTS · Aug 5, 2024 · e3669e8 · e3669e8
1 parent bcbd100
commit e3669e8
Show file tree

Hide file tree

Showing 5 changed files with 104 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -2,11 +2,46 @@
 Code for running GitLab CI/CD jobs on cloud.gov or another CloudFoundry based
 PaaS.
 
+* [Differences from other GitLab Runner executor types](#differences-from-other-gitlab-runner-executor-types)
 * [How it works](#how-it-works)
 * [Deploying](#deploying)
 * [Troubleshooting](#troubleshooting)
 * [Design Decisions](#design-decisions)
 
+## Differences from other GitLab Runner executor types
+
+The goal of this runner executor is to support most CI/CD use cases without
+needing to modify your `.gitlab-ci.yml`. Please note the current limitiations
+and differences in comparison to the Docker executor:
+
+* __No shared filesystem or direct exec capability__ - Some
+  executors can share filesystems between containers and directly execute processes
+  in containers. CloudFoundry does not support sharing filesystems and only supports
+  use of SSH to execute into a running container. This runner attempts to transparently
+  work around these limitations where possible. Your CI job may require significant
+  modification if it relies on either of these features.
+* __Use CI_SERVICE_alias for service DNS names__ - Ephemeral networks like the Docker networks
+  used for the Docker executor are not available in CloudFoundry. This means
+  that each service you create lives in a common internal DNS namespace with
+  other services in the same CloudFoundry space. The cloud.gov runner populates
+  a `CI_SERCVCE_<alias>` variable for each service defined in a job. Here is an
+  example snippet of GitLab CI Yaml showing the definition of a service and how
+  the job steps can then connect to that service:
+  ~~~yaml
+  # Start a HTTP "echo" service and then send a request to it
+  echo-test-job:
+    stage: test
+    image: ubuntu:jammy
+  services:
+  - name: http-https-echo:latest
+    # Note "echo" is the alias name, so the CI_SERVICE_alias variable key name
+    # for this service is CI_SERVICE_echo
+    alias: echo
+  script:
+  # Using the CI_SERVICE_alias to provide the FQDN of the service
+  - curl http://${CI_SERVICE_echo}:8080
+  ~~~
+
 ## How it works
 
 This is a custom executor borrowing ideas from https://docs.gitlab.com/runner/executors/custom.html.

diff --git a/manifest.yml b/manifest.yml
@@ -15,16 +15,15 @@ applications:
       - ((service_account_instance))
       - ((object_store_instance))
     env:
-      # These are used by .profile
       DEFAULT_JOB_IMAGE: ((default_job_image))
-      # Remaining vars are used directly by gitlab-runner register
-      # See gitlab-runner register --help for available vars
+      # See gitlab-runner register --help for available vars.
       CI_SERVER_TOKEN: ((ci_server_token))
       CI_SERVER_URL: ((ci_server_url))
       DOCKER_HUB_USER: ((docker_hub_user))
       DOCKER_HUB_TOKEN: ((docker_hub_token))
       RUNNER_EXECUTOR: ((runner_executor))
       RUNNER_NAME: ((runner_name))
+      OBJECT_STORE_INSTANCE: ((object_store_instance))
       # Remaining runner configuration is generally static. In order to surface
       # the entire configuration input, we are using envvars for all of it.
       RUNNER_BUILDS_DIR: "/tmp/build"
@@ -39,4 +38,4 @@ applications:
       # https://docs.gitlab.com/runner/faq/#enable-debug-logging-mode
       # and ensuring job logs are removed to avoid leaking secrets.
       RUNNER_DEBUG: "false"
-      OBJECT_STORE_INSTANCE: ((object_store_instance))
+
diff --git a/runner/cf-driver/base.sh b/runner/cf-driver/base.sh
@@ -10,12 +10,12 @@ CONTAINER_ID="glrw-r$CUSTOM_ENV_CI_RUNNER_ID-p$CUSTOM_ENV_CI_PROJECT_ID-c$CUSTOM
 # Set a fallback if not set but complain
 if [ -z "$DEFAULT_JOB_IMAGE" ]; then
     DEFAULT_JOB_IMAGE="ubuntu:latest"
-    echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}"
+    echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}" 1>&2
 fi
 
 # Complain if no Docker Hub credentials so we aren't bad neighbors
 if [ -z "$DOCKER_HUB_USER" ] || [ -z "$DOCKER_HUB_TOKEN" ]; then
-    echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting."
+    echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting." 1>&2
 fi
 
 # Use a custom image if provided, else fallback to configured default

diff --git a/runner/cf-driver/prepare.sh b/runner/cf-driver/prepare.sh
@@ -3,9 +3,9 @@
 set -euo pipefail
 
 # trap any error, and mark it as a system failure.
-# Also cleans up TMPVARFILE (set in create_temporary_varfile).
-trap 'rm -f "$TMPVARFILE"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR
-trap 'rm -f "$TMPVARFILE"' EXIT
+# Also cleans up TMPMANIFEST(set in create_temporary_manifest).
+trap 'rm -f "$TMPMANIFEST"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR
+trap 'rm -f "$TMPMANIFEST"' EXIT
 
 # Prepare a runner executor application in CloudFoundry
 
@@ -16,18 +16,6 @@ if [ -z "${WORKER_MEMORY-}" ]; then
     WORKER_MEMORY="512M"
 fi
 
-create_temporary_varfile () {
-    # A less leak-prone way to share secrets into the worker which will not
-    # be able to parse VCAP_CONFIGURATION
-    TMPVARFILE=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX)
-
-    for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do
-        echo "$v: \"$v\"" >> "$TMPVARFILE"
-    done
-
-    echo "[cf-driver] [DEBUG] Added $(wc -l "$TMPVARFILE") lines to $TMPVARFILE"
-}
-
 get_registry_credentials () {
     image_name="$1"
 
@@ -51,6 +39,28 @@ get_registry_credentials () {
     fi
 }
 
+create_temporary_manifest () {
+    # A less leak-prone way to share secrets into the worker which will not
+    # be able to parse VCAP_CONFIGURATION
+    TMPMANIFEST=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX)
+    chmod 600 "$TMPMANIFEST"
+    cat "${currentDir}/worker-manifest.yml" > "$TMPMANIFEST"
+
+    # Align additional environment variables with YAML at end of source manifest
+    local padding="      "
+
+    for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do
+        echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST"
+    done
+
+    # Add any CI_SERVICE_x variables populated by start_service()
+    for v in "${!CI_SERVICE_@}"; do
+        echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST"
+    done
+
+    echo "[cf-driver] [DEBUG] $(wc -l < "$TMPMANIFEST") lines in $TMPMANIFEST"
+}
+
 start_container () {
     container_id="$1"
     image_name="$CUSTOM_ENV_CI_JOB_IMAGE"
@@ -62,9 +72,8 @@ start_container () {
 
     push_args=(
        "$container_id"
-       -f "${currentDir}/worker-manifest.yml"
+       -f "$TMPMANIFEST"
        -m "$WORKER_MEMORY"
-       --vars-file "$TMPVARFILE"
        --docker-image "$image_name"
     )
 
@@ -118,7 +127,32 @@ start_service () {
 
     # TODO - Figure out how to handle non-global memory definition
     cf push "${push_args[@]}"
+
+    # Map route and export a FQDN. We assume apps.internal as the domain.
     cf map-route "$container_id" apps.internal --hostname "$container_id"
+    export "CI_SERVICE_${alias_name}"="${container_id}.apps.internal"
+}
+
+start_services () {
+    container_id_base="$1"
+    ci_job_services="$2"
+
+    if [ -z "$ci_job_services" ]; then
+       echo "[cf-driver] No services defined in ci_job_services - Skipping service startup"
+       return
+    fi
+
+    for l in $(echo "$ci_job_services" | jq -rc '.[]'); do
+        # Using jq -er to fail of alias or name are not found
+        alias_name=$(echo "$l" | jq -er '.alias | select(.)')
+        container_id="${container_id_base}-svc-${alias_name}"
+        image_name=$(echo "$l" | jq -er '.name | select(.)')
+        # Using jq -r to allow entrypoint and command to be empty
+        container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)')
+        container_command=$(echo "$l" | jq -r '.command | select(.)')
+
+        start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command"
+    done
 }
 
 allow_access_to_service () {
@@ -138,25 +172,17 @@ allow_access_to_service () {
         --protocol "$protocol" --port "$ports"
 }
 
-start_services () {
+allow_access_to_services () {
     container_id_base="$1"
     ci_job_services="$2"
 
     if [ -z "$ci_job_services" ]; then
-       echo "[cf-driver] No services defined in ci_job_services - Skipping service startup"
+       echo "[cf-driver] No services defined in ci_job_services - Skipping service allowance"
        return
     fi
 
     for l in $(echo "$ci_job_services" | jq -rc '.[]'); do
-        # Using jq -er to fail of alias or name are not found
-        alias_name=$(echo "$l" | jq -er '.alias | select(.)')
         container_id="${container_id_base}-svc-${alias_name}"
-        image_name=$(echo "$l" | jq -er '.name | select(.)')
-        # Using jq -r to allow entrypoint and command to be empty
-        container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)')
-        container_command=$(echo "$l" | jq -r '.command | select(.)')
-
-        start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command"
         allow_access_to_service "$container_id_base" "$container_id"
     done
 }
@@ -188,18 +214,24 @@ install_dependencies () {
                                ln -s /usr/bin/gitlab-runner-helper /usr/bin/gitlab-runner'
 }
 
-echo "[cf-driver] Preparing environment variables for $CONTAINER_ID"
-create_temporary_varfile
+if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then
+    echo "[cf-driver] Starting services"
+    start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
+fi
+
+echo "[cf-driver] Preparing manifest for $CONTAINER_ID"
+create_temporary_manifest
 
 echo "[cf-driver] Starting $CONTAINER_ID with image $CUSTOM_ENV_CI_JOB_IMAGE"
 start_container "$CONTAINER_ID"
 
 echo "[cf-driver] Installing dependencies into $CONTAINER_ID"
 install_dependencies "$CONTAINER_ID"
 
+# Allowing access last so services and the worker are all present
 if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then
-    echo "[cf-driver] Starting services"
-    start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
+    echo "[cf-driver] Enabling access from worker to services"
+    allow_access_to_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
 fi
 
 echo "[cf-driver] $CONTAINER_ID preparation complete"
diff --git a/runner/cf-driver/worker-manifest.yml b/runner/cf-driver/worker-manifest.yml
@@ -9,11 +9,4 @@ applications:
       RUNNER_BUILDS_DIR: "/tmp/build"
       RUNNER_CACHE_DIR: "/tmp/cache"
       CACHE_SHARED: "true"
-      # The following are passed through using a temporary vars-file in prepare.sh
-      CACHE_TYPE: ((CACHE_TYPE))
-      CACHE_PATH: ((RUNNER_NAME))
-      CACHE_S3_SERVER_ADDRESS: ((CACHE_S3_SERVER_ADDRESS))
-      CACHE_S3_BUCKET_LOCATION: ((CACHE_S3_BUCKET_LOCATION))
-      CACHE_S3_BUCKET_NAME: ((CACHE_S3_BUCKET_NAME))
-      CACHE_S3_ACCESS_KEY: ((CACHE_S3_ACCESS_KEY))
-      CACHE_S3_SECRET_KEY: ((CACHE_S3_SECRET_KEY))
+      # Additional items may be added below this point in prepare.sh