Adds dd-ansible-runner check and role

This adds a simple datadog check that fails if it finds any flag files for
failing ansible-runner environments.  This depends on PR BonnyCI#363
but should pass OK if it lands before that merges.

Related-Issue: BonnyCI/projman#224

Signed-off-by: Adam Gandelman <[email protected]>

bastion.yml

@@ -36,6 +36,9 @@
             - notifempty
       when: secrets is defined
 
+    - role: dd-ansible-runner
+      tags:
+        - monitoring
     - role: ansible-runner
       ansible_runner_minute: "*/15"
       ansible_runner_virtualenv: /opt/ansible

roles/dd-ansible-runner/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+dd_ansible_runner_failing_dir: /var/www/html/cron-logs/failing/

roles/dd-ansible-runner/files/ansible_runner.py

@@ -0,0 +1,30 @@
+
+import os
+
+from checks import AgentCheck
+
+
+class ZuulServerCheck(AgentCheck):
+
+    SERVICE_CHECK_NAME = 'ansible_runner.failing_environments'
+
+    def check(self, instance):
+        if 'failing_dir' not in instance:
+            self.log.info("Skipping instance, no failing_dir found")
+            return
+
+        failing_dir = instance['failing_dir']
+
+        if os.path.isdir(failing_dir):
+            failing_files = os.listdir(failing_dir)
+            if failing_files:
+                msg = 'ansible-runner envs failing: %s' % \
+                       ', '.join(failing_files)
+                self.service_check(self.SERVICE_CHECK_NAME,
+                                   AgentCheck.CRITICAL,
+                                   message=msg)
+        else:
+            self.service_check(self.SERVICE_CHECK_NAME,
+                               AgentCheck.OK,
+                               message="No failing ansible-runner envs found",
+                               tags=service_check_tags)

roles/dd-ansible-runner/meta/main.yml

@@ -0,0 +1,9 @@
+---
+dependencies:
+  - role: dd-checks
+
+    datadog_checks:
+      ansible_runner:
+        instances:
+          - failing_dir: "{{ dd_ansible_runner_failing_dir }}"
+

roles/dd-ansible-runner/tasks/main.yml

@@ -0,0 +1,5 @@
+- name: install ansible-runner check
+  copy:
+    src: ansible_runner.py
+    dest: "{{ datadog_file_dir }}/ansible_runner.py"
+  notify: restart dd-agent