diff --git a/README.md b/README.md index f561129..7036f3e 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,46 @@ Code for running GitLab CI/CD jobs on cloud.gov or another CloudFoundry based PaaS. +* [Differences from other GitLab Runner executor types](#differences-from-other-gitlab-runner-executor-types) * [How it works](#how-it-works) * [Deploying](#deploying) * [Troubleshooting](#troubleshooting) * [Design Decisions](#design-decisions) +## Differences from other GitLab Runner executor types + +The goal of this runner executor is to support most CI/CD use cases without +needing to modify your `.gitlab-ci.yml`. Please note the current limitiations +and differences in comparison to the Docker executor: + +* __No shared filesystem or direct exec capability__ - Some + executors can share filesystems between containers and directly execute processes + in containers. CloudFoundry does not support sharing filesystems and only supports + use of SSH to execute into a running container. This runner attempts to transparently + work around these limitations where possible. Your CI job may require significant + modification if it relies on either of these features. +* __Use CI_SERVICE_alias for service DNS names__ - Ephemeral networks like the Docker networks + used for the Docker executor are not available in CloudFoundry. This means + that each service you create lives in a common internal DNS namespace with + other services in the same CloudFoundry space. The cloud.gov runner populates + a `CI_SERCVCE_` variable for each service defined in a job. Here is an + example snippet of GitLab CI Yaml showing the definition of a service and how + the job steps can then connect to that service: + ~~~yaml + # Start a HTTP "echo" service and then send a request to it + echo-test-job: + stage: test + image: ubuntu:jammy + services: + - name: http-https-echo:latest + # Note "echo" is the alias name, so the CI_SERVICE_alias variable key name + # for this service is CI_SERVICE_echo + alias: echo + script: + # Using the CI_SERVICE_alias to provide the FQDN of the service + - curl http://${CI_SERVICE_echo}:8080 + ~~~ + ## How it works This is a custom executor borrowing ideas from https://docs.gitlab.com/runner/executors/custom.html. diff --git a/manifest.yml b/manifest.yml index 447d6de..fba12ef 100644 --- a/manifest.yml +++ b/manifest.yml @@ -15,16 +15,15 @@ applications: - ((service_account_instance)) - ((object_store_instance)) env: - # These are used by .profile DEFAULT_JOB_IMAGE: ((default_job_image)) - # Remaining vars are used directly by gitlab-runner register - # See gitlab-runner register --help for available vars + # See gitlab-runner register --help for available vars. CI_SERVER_TOKEN: ((ci_server_token)) CI_SERVER_URL: ((ci_server_url)) DOCKER_HUB_USER: ((docker_hub_user)) DOCKER_HUB_TOKEN: ((docker_hub_token)) RUNNER_EXECUTOR: ((runner_executor)) RUNNER_NAME: ((runner_name)) + OBJECT_STORE_INSTANCE: ((object_store_instance)) # Remaining runner configuration is generally static. In order to surface # the entire configuration input, we are using envvars for all of it. RUNNER_BUILDS_DIR: "/tmp/build" @@ -39,4 +38,4 @@ applications: # https://docs.gitlab.com/runner/faq/#enable-debug-logging-mode # and ensuring job logs are removed to avoid leaking secrets. RUNNER_DEBUG: "false" - OBJECT_STORE_INSTANCE: ((object_store_instance)) + diff --git a/runner/cf-driver/base.sh b/runner/cf-driver/base.sh index 99e012a..adb4d59 100644 --- a/runner/cf-driver/base.sh +++ b/runner/cf-driver/base.sh @@ -10,12 +10,12 @@ CONTAINER_ID="glrw-r$CUSTOM_ENV_CI_RUNNER_ID-p$CUSTOM_ENV_CI_PROJECT_ID-c$CUSTOM # Set a fallback if not set but complain if [ -z "$DEFAULT_JOB_IMAGE" ]; then DEFAULT_JOB_IMAGE="ubuntu:latest" - echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}" + echo "WARNING: DEFAULT_JOB_IMAGE not set! Falling back to ${DEFAULT_JOB_IMAGE}" 1>&2 fi # Complain if no Docker Hub credentials so we aren't bad neighbors if [ -z "$DOCKER_HUB_USER" ] || [ -z "$DOCKER_HUB_TOKEN" ]; then - echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting." + echo "WARNING: Docker Hub credentials not set! Falling back to public access which could result in rate limiting." 1>&2 fi # Use a custom image if provided, else fallback to configured default diff --git a/runner/cf-driver/prepare.sh b/runner/cf-driver/prepare.sh index e99f438..1d2d0ec 100755 --- a/runner/cf-driver/prepare.sh +++ b/runner/cf-driver/prepare.sh @@ -3,9 +3,9 @@ set -euo pipefail # trap any error, and mark it as a system failure. -# Also cleans up TMPVARFILE (set in create_temporary_varfile). -trap 'rm -f "$TMPVARFILE"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR -trap 'rm -f "$TMPVARFILE"' EXIT +# Also cleans up TMPMANIFEST(set in create_temporary_manifest). +trap 'rm -f "$TMPMANIFEST"; exit $SYSTEM_FAILURE_EXIT_CODE' ERR +trap 'rm -f "$TMPMANIFEST"' EXIT # Prepare a runner executor application in CloudFoundry @@ -16,18 +16,6 @@ if [ -z "${WORKER_MEMORY-}" ]; then WORKER_MEMORY="512M" fi -create_temporary_varfile () { - # A less leak-prone way to share secrets into the worker which will not - # be able to parse VCAP_CONFIGURATION - TMPVARFILE=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX) - - for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do - echo "$v: \"$v\"" >> "$TMPVARFILE" - done - - echo "[cf-driver] [DEBUG] Added $(wc -l "$TMPVARFILE") lines to $TMPVARFILE" -} - get_registry_credentials () { image_name="$1" @@ -51,6 +39,28 @@ get_registry_credentials () { fi } +create_temporary_manifest () { + # A less leak-prone way to share secrets into the worker which will not + # be able to parse VCAP_CONFIGURATION + TMPMANIFEST=$(mktemp /tmp/gitlab-runner-worker-manifest.XXXXXXXXXX) + chmod 600 "$TMPMANIFEST" + cat "${currentDir}/worker-manifest.yml" > "$TMPMANIFEST" + + # Align additional environment variables with YAML at end of source manifest + local padding=" " + + for v in RUNNER_NAME CACHE_TYPE CACHE_S3_SERVER_ADDRESS CACHE_S3_BUCKET_LOCATION CACHE_S3_BUCKET_NAME CACHE_S3_ACCESS_KEY CACHE_S3_SECRET_KEY; do + echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST" + done + + # Add any CI_SERVICE_x variables populated by start_service() + for v in "${!CI_SERVICE_@}"; do + echo "${padding}${v}: \"${!v}\"" >> "$TMPMANIFEST" + done + + echo "[cf-driver] [DEBUG] $(wc -l < "$TMPMANIFEST") lines in $TMPMANIFEST" +} + start_container () { container_id="$1" image_name="$CUSTOM_ENV_CI_JOB_IMAGE" @@ -62,9 +72,8 @@ start_container () { push_args=( "$container_id" - -f "${currentDir}/worker-manifest.yml" + -f "$TMPMANIFEST" -m "$WORKER_MEMORY" - --vars-file "$TMPVARFILE" --docker-image "$image_name" ) @@ -118,7 +127,32 @@ start_service () { # TODO - Figure out how to handle non-global memory definition cf push "${push_args[@]}" + + # Map route and export a FQDN. We assume apps.internal as the domain. cf map-route "$container_id" apps.internal --hostname "$container_id" + export "CI_SERVICE_${alias_name}"="${container_id}.apps.internal" +} + +start_services () { + container_id_base="$1" + ci_job_services="$2" + + if [ -z "$ci_job_services" ]; then + echo "[cf-driver] No services defined in ci_job_services - Skipping service startup" + return + fi + + for l in $(echo "$ci_job_services" | jq -rc '.[]'); do + # Using jq -er to fail of alias or name are not found + alias_name=$(echo "$l" | jq -er '.alias | select(.)') + container_id="${container_id_base}-svc-${alias_name}" + image_name=$(echo "$l" | jq -er '.name | select(.)') + # Using jq -r to allow entrypoint and command to be empty + container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)') + container_command=$(echo "$l" | jq -r '.command | select(.)') + + start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command" + done } allow_access_to_service () { @@ -138,25 +172,17 @@ allow_access_to_service () { --protocol "$protocol" --port "$ports" } -start_services () { +allow_access_to_services () { container_id_base="$1" ci_job_services="$2" if [ -z "$ci_job_services" ]; then - echo "[cf-driver] No services defined in ci_job_services - Skipping service startup" + echo "[cf-driver] No services defined in ci_job_services - Skipping service allowance" return fi for l in $(echo "$ci_job_services" | jq -rc '.[]'); do - # Using jq -er to fail of alias or name are not found - alias_name=$(echo "$l" | jq -er '.alias | select(.)') container_id="${container_id_base}-svc-${alias_name}" - image_name=$(echo "$l" | jq -er '.name | select(.)') - # Using jq -r to allow entrypoint and command to be empty - container_entrypoint=$(echo "$l" | jq -r '.entrypoint | select(.)') - container_command=$(echo "$l" | jq -r '.command | select(.)') - - start_service "$alias_name" "$container_id" "$image_name" "$container_entrypoint" "$container_command" allow_access_to_service "$container_id_base" "$container_id" done } @@ -188,8 +214,13 @@ install_dependencies () { ln -s /usr/bin/gitlab-runner-helper /usr/bin/gitlab-runner' } -echo "[cf-driver] Preparing environment variables for $CONTAINER_ID" -create_temporary_varfile +if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then + echo "[cf-driver] Starting services" + start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES" +fi + +echo "[cf-driver] Preparing manifest for $CONTAINER_ID" +create_temporary_manifest echo "[cf-driver] Starting $CONTAINER_ID with image $CUSTOM_ENV_CI_JOB_IMAGE" start_container "$CONTAINER_ID" @@ -197,9 +228,10 @@ start_container "$CONTAINER_ID" echo "[cf-driver] Installing dependencies into $CONTAINER_ID" install_dependencies "$CONTAINER_ID" +# Allowing access last so services and the worker are all present if [ -n "$CUSTOM_ENV_CI_JOB_SERVICES" ]; then - echo "[cf-driver] Starting services" - start_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES" + echo "[cf-driver] Enabling access from worker to services" + allow_access_to_services "$CONTAINER_ID" "$CUSTOM_ENV_CI_JOB_SERVICES" fi echo "[cf-driver] $CONTAINER_ID preparation complete" diff --git a/runner/cf-driver/worker-manifest.yml b/runner/cf-driver/worker-manifest.yml index 64ecbe2..4ce6507 100644 --- a/runner/cf-driver/worker-manifest.yml +++ b/runner/cf-driver/worker-manifest.yml @@ -9,11 +9,4 @@ applications: RUNNER_BUILDS_DIR: "/tmp/build" RUNNER_CACHE_DIR: "/tmp/cache" CACHE_SHARED: "true" - # The following are passed through using a temporary vars-file in prepare.sh - CACHE_TYPE: ((CACHE_TYPE)) - CACHE_PATH: ((RUNNER_NAME)) - CACHE_S3_SERVER_ADDRESS: ((CACHE_S3_SERVER_ADDRESS)) - CACHE_S3_BUCKET_LOCATION: ((CACHE_S3_BUCKET_LOCATION)) - CACHE_S3_BUCKET_NAME: ((CACHE_S3_BUCKET_NAME)) - CACHE_S3_ACCESS_KEY: ((CACHE_S3_ACCESS_KEY)) - CACHE_S3_SECRET_KEY: ((CACHE_S3_SECRET_KEY)) + # Additional items may be added below this point in prepare.sh