ansible-runner: Drop flag files for failing environments

This updates ansible-runner to drop empty files in a known
directory for environments that are failing to complete their
playbooks. The files are named after the failing environment.
Upon successful ansible run, these files are cleaned up if
they exist.

This'll allow us to set a datadog check that fails when any
files exist here.  Our current datadog check is broken and
relies on log scraping and doesn't really work with multi-env
bastions.

Related-Issue: BonnyCI/projman#224

Signed-off-by: Adam Gandelman <[email protected]>

roles/ansible-runner/files/usr/local/bin/ansible-runner

@@ -1,6 +1,8 @@
 #!/bin/bash -eu
 # Runs system wide ansible configured in /etc
 
+set -o pipefail
+
 env=$1
 shift
 if [ -f /etc/default/"$env" ]; then
@@ -10,6 +12,7 @@ fi
 envs_root=${ANSIBLE_RUNNER_ENV_ROOT:-/opt/source}
 ansible_venv=${ANSIBLE_RUNNER_VENV:-/opt/ansible}
 log_dir=${ANSIBLE_RUNNER_LOG_DIR:-/var/www/html/cron-logs/$env/}
+failing_dir=${ANSIBLE_RUNNER_FAILING_DIR:-/var/www/html/cron-logs/failing/}
 ssh_user=${ANSIBLE_SSH_USER:-""}
 force_runner=false
 ansible_options=""
@@ -60,7 +63,14 @@ fi
 if [ -n "$ssh_user" ]; then
     ansible_options="$ansible_options -e ansible_ssh_user=$ssh_user"
 fi
-"$ansible_venv"/bin/ansible-playbook -vv -i "$ANSIBLE_INVENTORY" $ansible_options "$ANSIBLE_PLAYBOOK" 2>&1 | tee -a "$logfile"
+
+if ! "$ansible_venv"/bin/ansible-playbook -vv -i "$ANSIBLE_INVENTORY" $ansible_options "$ANSIBLE_PLAYBOOK" 2>&1 | tee -a "$logfile"; then
+  mkdir -p "$failing_dir"
+  touch "${failing_dir}/${env}"
+else
+  rm -rf "${failing_dir}/${env}"
+fi
+
 date 2>&1 | tee -a "$logfile"
 
 # Remove > 10 days old

tests/files/ansible_runner_test.yml

@@ -1,7 +1,9 @@
 ---
-- name: Write a test file
+- name: A test playbook
   hosts: localhost
   tasks:
+    - fail:
+      when: fail_test_playbook is defined
     - copy:
         dest: %%TEST_FILE_PATH%%
         content: %%TEST_FILE_CONTENT%%

tests/test-ansible-runner.sh

@@ -7,6 +7,7 @@ export test_env="runner_test"
 runner_path="roles/ansible-runner/files/usr/local/bin/ansible-runner"
 test_file="$tmpdir/ansible_runner_test_file"
 test_log_dir="$tmpdir/logs"
+fail_file_dir="$tmpdir/failing"
 repo="$tmpdir/ansible_runner_env_repo"
 repo_playbook="$repo/ansible_runner_test.yml"
 repo_checkout="$tmpdir/runner_test"
@@ -26,9 +27,10 @@ function run_runner() {
   ANSIBLE_RUNNER_ENV_ROOT="$tmpdir" \
   ANSIBLE_RUNNER_VENV="$test_venv" \
   ANSIBLE_RUNNER_LOG_DIR="$test_log_dir" \
+  ANSIBLE_RUNNER_FAILING_DIR="$fail_file_dir" \
   ANSIBLE_PLAYBOOK=$(basename "$repo_playbook") \
   ANSIBLE_INVENTORY=/dev/null \
-  "$runner_path" runner_test
+  "$runner_path" runner_test "$@"
 }
 
 function commit_requirement() {
@@ -99,6 +101,15 @@ function assert_contents_and_logging() {
     echo "OK"
 }
 
+function assert_fail_file() {
+    if [[ -e "$fail_file_dir"/runner_test ]]; then
+        echo "Fail file exists at $fail_file_dir/runner_test"
+        return 0
+    else
+        echo "Fail file not found at $fail_file_dir/runner_test"
+        return 1
+    fi
+}
 
 # Initialize the upstream environment repo from which ansible-runner will pull
 # and run
@@ -126,7 +137,16 @@ assert_contents_and_logging "$test_file" "bar" "$test_log_dir" "Long Live Anne B
 
 # Add another requirement to the upstream requirements
 commit_requirement "$repo"/requirements.txt iso8601
-
 run_runner
-
+# Ensure requirement has been successfully installed in venv
 assert_requirement iso8601 "$test_venv"
+
+# Run the playbook with the fail option
+run_runner "-e fail_test_playbook=true"
+# Ensure ansible-runner created a flag file for the failing env
+assert_fail_file
+
+# Run the playbook once more without fail option
+run_runner
+# Ensure the flag file is cleaned up
+! assert_fail_file