Add CI_SERVICE_alias environment variables to make referencing the FQDN of services fun!

manifest.yml

@@ -15,16 +15,15 @@ applications:
       - ((service_account_instance))
       - ((object_store_instance))
     env:
-      # These are used by .profile
       DEFAULT_JOB_IMAGE: ((default_job_image))
-      # Remaining vars are used directly by gitlab-runner register
-      # See gitlab-runner register --help for available vars
+      # See gitlab-runner register --help for available vars.
       CI_SERVER_TOKEN: ((ci_server_token))
       CI_SERVER_URL: ((ci_server_url))
       DOCKER_HUB_USER: ((docker_hub_user))
       DOCKER_HUB_TOKEN: ((docker_hub_token))
       RUNNER_EXECUTOR: ((runner_executor))
       RUNNER_NAME: ((runner_name))
+      OBJECT_STORE_INSTANCE: ((object_store_instance))
       # Remaining runner configuration is generally static. In order to surface
       # the entire configuration input, we are using envvars for all of it.
       RUNNER_BUILDS_DIR: "/tmp/build"
@@ -39,4 +38,4 @@ applications:
       # https://docs.gitlab.com/runner/faq/#enable-debug-logging-mode
       # and ensuring job logs are removed to avoid leaking secrets.
       RUNNER_DEBUG: "false"
-      OBJECT_STORE_INSTANCE: ((object_store_instance))
+

runner/cf-driver/base.sh

@@ -10,12 +10,12 @@ CONTAINER_ID="glrw-r$CUSTOM_ENV_CI_RUNNER_ID-p$CUSTOM_ENV_CI_PROJECT_ID-c$CUSTOM
 # Set a fallback if not set but complain
 if [ -z "$DEFAULT_JOB_IMAGE" ]; then
     DEFAULT_JOB_IMAGE="ubuntu:latest"
-    echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}"
+    echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}" 1>&2
 fi
 
 # Complain if no Docker Hub credentials so we aren't bad neighbors
 if [ -z "$DOCKER_HUB_USER" ] || [ -z "$DOCKER_HUB_TOKEN" ]; then
-    echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting."
+    echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting." 1>&2
 fi
 
 # Use a custom image if provided, else fallback to configured default

runner/cf-driver/prepare.sh

@@ -3,9 +3,9 @@
 set -euo pipefail
 
 # trap any error, and mark it as a system failure.
-# Also cleans up TMPVARFILE (set in create_temporary_varfile).
-trap 'rm -f "$TMPVARFILE"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR
-trap 'rm -f "$TMPVARFILE"' EXIT
+# Also cleans up TMPMANIFEST(set in create_temporary_manifest).
+trap 'rm -f "$TMPMANIFEST"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR
+trap 'rm -f "$TMPMANIFEST"' EXIT
 
 # Prepare a runner executor application in CloudFoundry
 
@@ -16,18 +16,6 @@ if [ -z "${WORKER_MEMORY-}" ]; then
     WORKER_MEMORY="512M"
 fi
 
-create_temporary_varfile () {
-    # A less leak-prone way to share secrets into the worker which will not
-    # be able to parse VCAP_CONFIGURATION
-    TMPVARFILE=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX)
-
-    for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do
-        echo "$v: \"$v\"" >> "$TMPVARFILE"
-    done
-
-    echo "[cf-driver] [DEBUG] Added $(wc -l "$TMPVARFILE") lines to $TMPVARFILE"
-}
-
 get_registry_credentials () {
     image_name="$1"
 
@@ -51,6 +39,28 @@ get_registry_credentials () {
     fi
 }
 
+create_temporary_manifest () {
+    # A less leak-prone way to share secrets into the worker which will not
+    # be able to parse VCAP_CONFIGURATION
+    TMPMANIFEST=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX)
+    chmod 600 "$TMPMANIFEST"
+    cat "${currentDir}/worker-manifest.yml" > "$TMPMANIFEST"
+
+    # Align additional environment variables with YAML at end of source manifest
+    local padding="      "
+
+    for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do
+        echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST"
+    done
+
+    # Add any CI_SERVICE_x variables populated by start_service()
+    for v in "${!CI_SERVICE_@}"; do
+        echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST"
+    done
+
+    echo "[cf-driver] [DEBUG] $(wc -l < "$TMPMANIFEST") lines in $TMPMANIFEST"
+}
+
 start_container () {
     container_id="$1"
     image_name="$CUSTOM_ENV_CI_JOB_IMAGE"
@@ -62,9 +72,8 @@ start_container () {
 
     push_args=(
        "$container_id"
-       -f "${currentDir}/worker-manifest.yml"
+       -f "$TMPMANIFEST"
        -m "$WORKER_MEMORY"
-       --vars-file "$TMPVARFILE"
        --docker-image "$image_name"
     )
 
@@ -118,7 +127,32 @@ start_service () {
 
     # TODO - Figure out how to handle non-global memory definition
     cf push "${push_args[@]}"
+
+    # Map route and export a FQDN. We assume apps.internal as the domain.
     cf map-route "$container_id" apps.internal --hostname "$container_id"
+    export "CI_SERVICE_${alias_name}"="${container_id}.apps.internal"
+}
+
+start_services () {
+    container_id_base="$1"
+    ci_job_services="$2"
+
+    if [ -z "$ci_job_services" ]; then
+       echo "[cf-driver] No services defined in ci_job_services - Skipping service startup"
+       return
+    fi
+
+    for l in $(echo "$ci_job_services" | jq -rc '.[]'); do
+        # Using jq -er to fail of alias or name are not found
+        alias_name=$(echo "$l" | jq -er '.alias | select(.)')
+        container_id="${container_id_base}-svc-${alias_name}"
+        image_name=$(echo "$l" | jq -er '.name | select(.)')
+        # Using jq -r to allow entrypoint and command to be empty
+        container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)')
+        container_command=$(echo "$l" | jq -r '.command | select(.)')
+
+        start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command"
+    done
 }
 
 allow_access_to_service () {
@@ -138,25 +172,17 @@ allow_access_to_service () {
         --protocol "$protocol" --port "$ports"
 }
 
-start_services () {
+allow_access_to_services () {
     container_id_base="$1"
     ci_job_services="$2"
 
     if [ -z "$ci_job_services" ]; then
-       echo "[cf-driver] No services defined in ci_job_services - Skipping service startup"
+       echo "[cf-driver] No services defined in ci_job_services - Skipping service allowance"
        return
     fi
 
     for l in $(echo "$ci_job_services" | jq -rc '.[]'); do
-        # Using jq -er to fail of alias or name are not found
-        alias_name=$(echo "$l" | jq -er '.alias | select(.)')
         container_id="${container_id_base}-svc-${alias_name}"
-        image_name=$(echo "$l" | jq -er '.name | select(.)')
-        # Using jq -r to allow entrypoint and command to be empty
-        container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)')
-        container_command=$(echo "$l" | jq -r '.command | select(.)')
-
-        start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command"
         allow_access_to_service "$container_id_base" "$container_id"
     done
 }
@@ -188,18 +214,24 @@ install_dependencies () {
                                ln -s /usr/bin/gitlab-runner-helper /usr/bin/gitlab-runner'
 }
 
-echo "[cf-driver] Preparing environment variables for $CONTAINER_ID"
-create_temporary_varfile
+if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then
+    echo "[cf-driver] Starting services"
+    start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
+fi
+
+echo "[cf-driver] Preparing manifest for $CONTAINER_ID"
+create_temporary_manifest
 
 echo "[cf-driver] Starting $CONTAINER_ID with image $CUSTOM_ENV_CI_JOB_IMAGE"
 start_container "$CONTAINER_ID"
 
 echo "[cf-driver] Installing dependencies into $CONTAINER_ID"
 install_dependencies "$CONTAINER_ID"
 
+# Allowing access last so services and the worker are all present
 if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then
-    echo "[cf-driver] Starting services"
-    start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
+    echo "[cf-driver] Enabling access from worker to services"
+    allow_access_to_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES"
 fi
 
 echo "[cf-driver] $CONTAINER_ID preparation complete"

runner/cf-driver/worker-manifest.yml

@@ -9,11 +9,4 @@ applications:
       RUNNER_BUILDS_DIR: "/tmp/build"
       RUNNER_CACHE_DIR: "/tmp/cache"
       CACHE_SHARED: "true"
-      # The following are passed through using a temporary vars-file in prepare.sh
-      CACHE_TYPE: ((CACHE_TYPE))
-      CACHE_PATH: ((RUNNER_NAME))
-      CACHE_S3_SERVER_ADDRESS: ((CACHE_S3_SERVER_ADDRESS))
-      CACHE_S3_BUCKET_LOCATION: ((CACHE_S3_BUCKET_LOCATION))
-      CACHE_S3_BUCKET_NAME: ((CACHE_S3_BUCKET_NAME))
-      CACHE_S3_ACCESS_KEY: ((CACHE_S3_ACCESS_KEY))
-      CACHE_S3_SECRET_KEY: ((CACHE_S3_SECRET_KEY))
+      # Additional items may be added below this point in prepare.sh