Merge pull request #173 from pyouroboros/develop

v1.1.1 Merge

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Change Log
 
+## [1.1.1](https://github.com/pyouroboros/ouroboros/tree/1.1.1) (2019-02-01)
+[Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.1.0...1.1.1)
+
+**Implemented enhancements:**
+
+- Support for adding an identifier \(hostname?\) to notifications [\#158](https://github.com/pyouroboros/ouroboros/issues/158)
+- Influx config data + ocd cleanup [\#162](https://github.com/pyouroboros/ouroboros/pull/162) [[cleanup](https://github.com/pyouroboros/ouroboros/labels/cleanup)] ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- add cli arg for cron [\#157](https://github.com/pyouroboros/ouroboros/pull/157) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+
+**Fixed bugs:**
+
+- Ouroboros does not respect MONITOR= [\#166](https://github.com/pyouroboros/ouroboros/issues/166)
+- Docker TLS over TCP connections [\#154](https://github.com/pyouroboros/ouroboros/issues/154) [[documentation](https://github.com/pyouroboros/ouroboros/labels/documentation)]
+- Patch/group 4 [\#169](https://github.com/pyouroboros/ouroboros/pull/169) [[cleanup](https://github.com/pyouroboros/ouroboros/labels/cleanup)] ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- Recheck properly for only non lists [\#164](https://github.com/pyouroboros/ouroboros/pull/164) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- add some missing passthrough info for restart [\#163](https://github.com/pyouroboros/ouroboros/pull/163) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+
+**Other Pull Requests**
+
+- v1.1.1 Merge [\#173](https://github.com/pyouroboros/ouroboros/pull/173) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- v1.1.1 to develop [\#172](https://github.com/pyouroboros/ouroboros/pull/172) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- Patch/group 3 [\#167](https://github.com/pyouroboros/ouroboros/pull/167) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+- Add hostname to the notifications [\#161](https://github.com/pyouroboros/ouroboros/pull/161) ([tlkamp](https://github.com/tlkamp))
+- Patch/group 2 [\#155](https://github.com/pyouroboros/ouroboros/pull/155) ([DirtyCajunRice](https://github.com/DirtyCajunRice))
+
 ## [1.1.0](https://github.com/pyouroboros/ouroboros/tree/1.1.0) (2019-01-26)
 [Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.0.0...1.1.0)
 

pyouroboros/__init__.py

@@ -1,2 +1,2 @@
-VERSION = "1.1.0"
+VERSION = "1.1.1"
 BRANCH = "master"

pyouroboros/config.py

@@ -1,17 +1,21 @@
+from os import environ
 from logging import getLogger
 from pyouroboros.logger import BlacklistFilter
 
 
 class Config(object):
     options = ['INTERVAL', 'PROMETHEUS', 'DOCKER_SOCKETS', 'MONITOR', 'IGNORE', 'LOG_LEVEL', 'PROMETHEUS_ADDR',
-               'PROMETHEUS_PORT', 'NOTIFIERS', 'REPO_USER', 'REPO_PASS', 'CLEANUP', 'RUN_ONCE', 'LATEST',
+               'PROMETHEUS_PORT', 'NOTIFIERS', 'REPO_USER', 'REPO_PASS', 'CLEANUP', 'RUN_ONCE', 'LATEST', 'CRON',
                'INFLUX_URL', 'INFLUX_PORT', 'INFLUX_USERNAME', 'INFLUX_PASSWORD', 'INFLUX_DATABASE', 'INFLUX_SSL',
-               'INFLUX_VERIFY_SSL', 'DATA_EXPORT', 'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS_VERIFY', 'LABELS_ONLY',
-               'DRY_RUN']
+               'INFLUX_VERIFY_SSL', 'DATA_EXPORT', 'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS', 'LABELS_ONLY',
+               'DRY_RUN', 'HOSTNAME', 'DOCKER_TLS_VERIFY']
 
+    hostname = environ.get('HOSTNAME')
     interval = 300
+    cron = None
     docker_sockets = 'unix://var/run/docker.sock'
-    docker_tls_verify = False
+    docker_tls = False
+    docker_tls_verify = True
     monitor = []
     ignore = []
     data_export = None
@@ -76,23 +80,27 @@ def config_blacklist(self):
     def parse(self):
         for option in Config.options:
             if self.environment_vars.get(option):
+                env_opt = self.environment_vars[option]
+                if isinstance(env_opt, str):
+                    # Clean out quotes, both single/double and whitespace
+                    env_opt = env_opt.strip("'").strip('"').strip(' ')
                 if option in ['INTERVAL', 'PROMETHEUS_PORT', 'INFLUX_PORT']:
                     try:
-                        opt = int(self.environment_vars[option])
+                        opt = int(env_opt)
                         setattr(self, option.lower(), opt)
                     except ValueError as e:
                         print(e)
                 elif option in ['LATEST', 'CLEANUP', 'RUN_ONCE', 'INFLUX_SSL', 'INFLUX_VERIFY_SSL', 'DRY_RUN',
-                                'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS_VERIFY', 'LABELS_ONLY']:
-                    if self.environment_vars[option].lower() in ['true', 'yes']:
+                                'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS', 'LABELS_ONLY', 'DOCKER_TLS_VERIFY']:
+                    if env_opt.lower() in ['true', 'yes']:
                         setattr(self, option.lower(), True)
-                    elif self.environment_vars[option].lower() in ['false', 'no']:
+                    elif env_opt.lower() in ['false', 'no']:
                         setattr(self, option.lower(), False)
                     else:
-                        self.logger.error('%s is not true/yes, nor false/no for %s. Assuming false',
-                                          self.environment_vars[option], option)
+                        self.logger.error('%s is not true/yes, nor false/no for %s. Assuming %s',
+                                          env_opt, option, getattr(self, option))
                 else:
-                    setattr(self, option.lower(), self.environment_vars[option])
+                    setattr(self, option.lower(), env_opt)
             elif vars(self.cli_args).get(option):
                 setattr(self, option.lower(), vars(self.cli_args).get(option))
 
@@ -106,9 +114,19 @@ def parse(self):
         for option in ['docker_sockets', 'notifiers', 'monitor', 'ignore']:
             if isinstance(getattr(self, option), str):
                 string_list = getattr(self, option)
-                setattr(self, option, [string.strip(' ').strip('"') for string in string_list.split(' ')])
+                setattr(self, option, [string for string in string_list.split(' ')])
 
         # Config sanity checks
+        if self.cron:
+            cron_times = self.cron.strip().split(' ')
+            if len(cron_times) != 5:
+                self.logger.error("Cron must be in cron syntax. e.g. * * * * * (5 places). Ignoring and using interval")
+                self.cron = None
+            else:
+                self.logger.info("Cron configuration is valid. Using Cron schedule %s", cron_times)
+                self.cron = cron_times
+                self.interval = None
+
         if self.data_export == 'influxdb' and not self.influx_database:
             self.logger.error("You need to specify an influx database if you want to export to influxdb. Disabling "
                               "influxdb data export.")
@@ -120,4 +138,11 @@ def parse(self):
             self.logger.warning("Dry run is designed to be ran with run once. Setting for you.")
             self.run_once = True
 
+        # Remove default config that is not used for cleaner logs
+        if self.data_export != 'prometheus':
+            self.prometheus_addr, self.prometheus_port = None, None
+
+        if self.data_export != 'influxdb':
+            self.influx_url, self.influx_port, self.influx_username, self.influx_password = None, None, None, None
+
         self.config_blacklist()

pyouroboros/dataexporters.py

@@ -24,9 +24,8 @@ def add(self, label, socket):
         elif self.config.data_export == "influxdb" and self.enabled:
             if label == "all":
                 self.logger.debug("Total containers updated %s", self.total_updated[socket])
-                self.influx.write_points(label, socket)
-            else:
-                self.influx.write_points(label, socket)
+
+            self.influx.write_points(label, socket)
 
     def set(self, socket):
         if self.config.data_export == "prometheus" and self.enabled:
@@ -109,6 +108,13 @@ def write_points(self, label, socket):
                 "tags": {'socket': clean_socket},
                 "time": now,
                 "fields": {}
+            },
+            {
+                "measurement": "Ouroboros",
+                "tags": {'configuration': self.config.hostname},
+                "time": now,
+                "fields": {key: (value if not isinstance(value, list) else ' '.join(value)) for key, value in
+                           vars(self.config).items() if key.upper() in self.config.options}
             }
         ]
         if label == "all":

pyouroboros/dockerclient.py

@@ -1,6 +1,7 @@
 from time import sleep
 from logging import getLogger
-from docker import DockerClient
+from docker import DockerClient, tls
+from os.path import isdir, isfile, join
 from docker.errors import DockerException, APIError, NotFound
 
 from pyouroboros.helpers import set_properties
@@ -10,14 +11,51 @@ class Docker(object):
     def __init__(self, socket, config, data_manager, notification_manager):
         self.config = config
         self.socket = socket
-        self.client = DockerClient(base_url=socket, tls=self.config.docker_tls_verify)
+        self.client = self.connect()
         self.data_manager = data_manager
         self.data_manager.total_updated[self.socket] = 0
         self.logger = getLogger()
         self.monitored = self.monitor_filter()
 
         self.notification_manager = notification_manager
 
+    def connect(self):
+        if self.config.docker_tls:
+            try:
+                cert_paths = {
+                    'cert_top_dir': '/etc/docker/certs.d/',
+                    'clean_socket': self.socket.split('//')[1]
+                }
+                cert_paths['cert_dir'] = join(cert_paths['cert_top_dir'], cert_paths['clean_socket'])
+                cert_paths['cert_files'] = {
+                    'client_cert': join(cert_paths['cert_dir'], 'client.cert'),
+                    'client_key': join(cert_paths['cert_dir'], 'client.key'),
+                    'ca_crt': join(cert_paths['cert_dir'], 'ca.crt')
+                }
+
+                if not isdir(cert_paths['cert_dir']):
+                    self.logger.error('%s is not a valid cert folder', cert_paths['cert_dir'])
+                    raise ValueError
+
+                for cert_file in cert_paths['cert_files'].values():
+                    if not isfile(cert_file):
+                        self.logger.error('%s does not exist', cert_file)
+                        raise ValueError
+
+                tls_config = tls.TLSConfig(
+                    ca_cert=cert_paths['cert_files']['ca_crt'],
+                    verify=cert_paths['cert_files']['ca_crt'] if self.config.docker_tls_verify else False,
+                    client_cert=(cert_paths['cert_files']['client_cert'], cert_paths['cert_files']['client_key'])
+                )
+                client = DockerClient(base_url=self.socket, tls=tls_config)
+            except ValueError:
+                self.logger.error('Invalid Docker TLS config for %s, reverting to unsecured', self.socket)
+                client = DockerClient(base_url=self.socket)
+        else:
+            client = DockerClient(base_url=self.socket)
+
+        return client
+
     def get_running(self):
         """Return running container objects list, except ouroboros itself"""
         running_containers = []
@@ -52,7 +90,8 @@ def monitor_filter(self):
                     monitored_containers.append(container)
                 else:
                     continue
-            elif not self.config.labels_only and self.config.monitor and container.name not in self.config.ignore:
+            elif not self.config.labels_only and self.config.monitor and container.name in self.config.monitor \
+                    and container.name not in self.config.ignore:
                     monitored_containers.append(container)
             elif not self.config.labels_only and container.name not in self.config.ignore:
                     monitored_containers.append(container)
@@ -144,12 +183,18 @@ def update_containers(self):
 
             # If current running container is running latest image
             if current_image.id != latest_image.id:
-                if container.name in ['ouroboros', 'ouroboros-updated']:
-                    self.update_self(old_container=container, new_image=latest_image, count=1)
-
                 updated_container_tuples.append(
                     (container, current_image, latest_image)
                 )
+
+                if container.name in ['ouroboros', 'ouroboros-updated']:
+                    self.data_manager.total_updated[self.socket] += 1
+                    self.data_manager.add(label=container.name, socket=self.socket)
+                    self.data_manager.add(label='all', socket=self.socket)
+                    self.notification_manager.send(container_tuples=updated_container_tuples,
+                                                   socket=self.socket, kind='update')
+                    self.update_self(old_container=container, new_image=latest_image, count=1)
+
                 self.logger.info('%s will be updated', container.name)
 
                 # Get container list to restart after update complete

pyouroboros/helpers.py

@@ -1,10 +1,18 @@
-from string import Template
-
-
 def set_properties(old, new, self_name=None):
     """Store object for spawning new container in place of the one with outdated image"""
     properties = {
         'name': self_name if self_name else old.name,
+        'hostname': old.attrs['Config']['Hostname'],
+        'user': old.attrs['Config']['User'],
+        'domainname': old.attrs['Config']['Domainname'],
+        'tty': old.attrs['Config']['Tty'],
+        'ports': None if not old.attrs['Config'].get('ExposedPorts') else [
+            (p.split('/')[0], p.split('/')[1]) for p in old.attrs['Config']['ExposedPorts'].keys()
+        ],
+        'volumes': None if not old.attrs['Config'].get('Volumes') else [
+            v for v in old.attrs['Config']['Volumes'].keys()
+        ],
+        'working_dir': old.attrs['Config']['WorkingDir'],
         'image': new.tags[0],
         'command': old.attrs['Config']['Cmd'],
         'host_config': old.attrs['HostConfig'],
@@ -14,12 +22,3 @@ def set_properties(old, new, self_name=None):
     }
 
     return properties
-
-
-NotificationTemplate = Template(
-    'Host Socket: ${HOST_SOCKET}\n'
-    'Containers Monitored: ${CONTAINERS_MONITORED}\n'
-    'Containers Updated: ${CONTAINERS_UPDATED}\n'
-    'Containers Updated This Pass: {CONTAINERS_THIS_PASS}'
-    '${CONTAINER_UPDATES}'
-)

pyouroboros/logger.py

@@ -6,10 +6,8 @@ class BlacklistFilter(Filter):
     Log filter for blacklisted tokens and passwords
     """
 
-    blacklisted_keys = ['repo_user', 'repo_pass', 'auth_json', 'webhook_urls', 'docker_sockets', 'pushover_user',
-                        'prometheus_addr', 'influx_username', 'influx_password', 'influx_url', 'pushover_token',
-                        'pushover_device', 'smtp_host', 'smtp_username', 'smtp_password', 'smtp_recipients',
-                        'smtp_from_email']
+    blacklisted_keys = ['repo_user', 'repo_pass', 'auth_json', 'docker_sockets', 'prometheus_addr',
+                        'influx_username', 'influx_password', 'influx_url', 'notifiers']
 
     def __init__(self, filteredstrings):
         super().__init__()

pyouroboros/notifiers.py

@@ -1,7 +1,7 @@
 import apprise
 
 from logging import getLogger
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timezone
 
 
 class NotificationManager(object):
@@ -32,18 +32,18 @@ def build_apprise(self):
 
         return apprise_obj
 
-    def send(self, container_tuples=None, socket=None, kind='update'):
+    def send(self, container_tuples=None, socket=None, kind='update', next_run=None):
         if kind == 'startup':
             now = datetime.now(timezone.utc).astimezone()
             title = f'Ouroboros has started'
             body_fields = [
+                f'Host: {self.config.hostname}',
                 f'Time: {now.strftime("%Y-%m-%d %H:%M:%S")}',
-                f'Next Run: {(now + timedelta(0, self.config.interval)).strftime("%Y-%m-%d %H:%M:%S")}'
-            ]
+                f'Next Run: {next_run}']
         else:
             title = 'Ouroboros has updated containers!'
             body_fields = [
-                f"Host Socket: {socket.split('//')[1]}",
+                f"Host/Socket: {self.config.hostname} / {socket.split('//')[1]}",
                 f"Containers Monitored: {self.data_manager.monitored_containers[socket]}",
                 f"Total Containers Updated: {self.data_manager.total_updated[socket]}",
                 f"Containers updated this pass: {len(container_tuples)}"
@@ -57,7 +57,7 @@ def send(self, container_tuples=None, socket=None, kind='update'):
                     ) for container, old_image, new_image in container_tuples
                 ]
             )
-        body = '\n'.join(body_fields)
+        body = '\r\n'.join(body_fields)
 
         if self.apprise.servers:
             self.apprise.notify(title=title, body=body)