diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6539e5cf..00000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -sudo: true -dist: xenial -language: python -python: - - 3.7.2 -services: - - docker -addons: - apt: - packages: - - docker-ce - -script: - - pip install flake8 && flake8 --max-line-length 120 *.py pyouroboros/ - - python3 ./ouroboros -l debug --run-once - -deploy: - - provider: pypi - user: pyouroboros - password: $PYPI_PASSWORD - skip_existing: true - on: - branch: master - - provider: script - script: bash deploy.sh - on: - branch: master - - provider: script - script: bash deploy.sh - on: - branch: develop \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index efcf7c7d..4b866691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Change Log +## [1.2.0](https://github.com/pyouroboros/ouroboros/tree/1.2.0) (2019-02-13) +[Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.1.2...1.2.0) + +**Implemented enhancements:** + +- Move "Interval container update" messages to debug log level [\#194](https://github.com/pyouroboros/ouroboros/issues/194) +- \[Feature Request\] Support for Swarm Services [\#178](https://github.com/pyouroboros/ouroboros/issues/178) +- Add Warning for label\_enable not set while using labels\_only [\#202](https://github.com/pyouroboros/ouroboros/pull/202) ([larsderidder](https://github.com/larsderidder)) + +**Fixed bugs:** + +- Change depends\_on logic [\#198](https://github.com/pyouroboros/ouroboros/issues/198) +- Containers relying upon network namespace of a container that gets updated breaks when the parent container is recreated [\#197](https://github.com/pyouroboros/ouroboros/issues/197) +- Exception when trying to update container with complex compose networks [\#196](https://github.com/pyouroboros/ouroboros/issues/196) +- Problem with network IPv4 address carry-over [\#193](https://github.com/pyouroboros/ouroboros/issues/193) +- Monitor Ignored Re-Address + jenkins cleanup [\#191](https://github.com/pyouroboros/ouroboros/pull/191) [[cleanup](https://github.com/pyouroboros/ouroboros/labels/cleanup)] ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + +**Closed issues:** + +- Remove legacy --latest [\#206](https://github.com/pyouroboros/ouroboros/issues/206) [[breaking change](https://github.com/pyouroboros/ouroboros/labels/breaking%20change)] [[cleanup](https://github.com/pyouroboros/ouroboros/labels/cleanup)] +- Add environment variables in Wiki [\#203](https://github.com/pyouroboros/ouroboros/issues/203) [[documentation](https://github.com/pyouroboros/ouroboros/labels/documentation)] +- Slack notifications via webhook not working [\#187](https://github.com/pyouroboros/ouroboros/issues/187) + +**Other Pull Requests** + +- v1.2.0 Merge [\#208](https://github.com/pyouroboros/ouroboros/pull/208) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- v1.2.0 to develop [\#207](https://github.com/pyouroboros/ouroboros/pull/207) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Patch/tag bug [\#205](https://github.com/pyouroboros/ouroboros/pull/205) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Patch/group 5 [\#201](https://github.com/pyouroboros/ouroboros/pull/201) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Fix bug in user defined network detection [\#200](https://github.com/pyouroboros/ouroboros/pull/200) ([nightvisi0n](https://github.com/nightvisi0n)) +- Adjust apscheduler logger [\#199](https://github.com/pyouroboros/ouroboros/pull/199) ([circa10a](https://github.com/circa10a)) +- Carry over network config [\#195](https://github.com/pyouroboros/ouroboros/pull/195) ([nightvisi0n](https://github.com/nightvisi0n)) +- Jenkins tweaks [\#192](https://github.com/pyouroboros/ouroboros/pull/192) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Swarm + Jenkins [\#188](https://github.com/pyouroboros/ouroboros/pull/188) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + ## [1.1.2](https://github.com/pyouroboros/ouroboros/tree/1.1.2) (2019-02-02) [Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.1.1...1.1.2) @@ -11,8 +46,11 @@ - cron documentation example update [\#182](https://github.com/pyouroboros/ouroboros/issues/182) [[documentation](https://github.com/pyouroboros/ouroboros/labels/documentation)] -***Other Pull Requests** -- v1.1.2 Merge [\#185](https://github.com/pyouroboros/ouroboros/pull/185) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +**Other Pull Requests** + +- v1.1.2 Merge [\#186](https://github.com/pyouroboros/ouroboros/pull/186) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- v1.1.2 to develop [\#183](https://github.com/pyouroboros/ouroboros/pull/183) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Fix default timezone [\#177](https://github.com/pyouroboros/ouroboros/pull/177) ([circa10a](https://github.com/circa10a)) ## [1.1.1](https://github.com/pyouroboros/ouroboros/tree/1.1.1) (2019-02-01) [Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.1.0...1.1.1) diff --git a/Dockerfile.arm b/Dockerfile.arm index d242cdb2..2d3e876c 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -6,8 +6,6 @@ ENV TZ UTC WORKDIR /app -COPY /qemu-arm-static /usr/bin/qemu-arm-static - COPY /requirements.txt /setup.py /ouroboros /README.md /app/ COPY /pyouroboros /app/pyouroboros diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 4ff6acbd..6b99bf36 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -6,8 +6,6 @@ ENV TZ UTC WORKDIR /app -COPY /qemu-aarch64-static /usr/bin/qemu-aarch64-static - COPY /requirements.txt /setup.py /ouroboros /README.md /app/ COPY /pyouroboros /app/pyouroboros diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..bd385e0f --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,138 @@ +pipeline { + agent none + environment { + DOCKER_REPO = "pyouroboros/ouroboros" + GIT_REPO = 'pyouroboros/ouroboros' + VERSION_FILE = "pyouroboros/__init__.py" + FLAKE_FILES = "ouroboros *.py pyouroboros/*.py" + TAG = "" + GIT_TOKEN = credentials('github-jenkins-token') + PYPI_CREDS = credentials('pypi-creds-dirtycajunrice') + } + stages { + stage('Flake8 + Run Once') { + agent { label 'amd64'} + steps { + sh """ + python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 ${FLAKE_FILES} + venv/bin/python -m pip install -r requirements.txt && venv/bin/python ouroboros --log-level debug --run-once + rm -rf venv/ + """ + script { + TAG = sh(returnStdout: true, script: 'grep -i version ${VERSION_FILE} | cut -d" " -f3 | tr -d \\"').trim() + } + } + } + stage('Docker Builds') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + parallel { + stage('amd64') { + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def image = docker.build("${DOCKER_REPO}:${TAG}-amd64") + image.push() + + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${DOCKER_REPO}:develop-amd64") + image.push() + } + } + } + } + stage('ARMv6') { + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def image = docker.build("${DOCKER_REPO}:${TAG}-arm", "-f Dockerfile.arm .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${DOCKER_REPO}:develop-arm", "-f Dockerfile.arm .") + image.push() + } + } + } + } + stage('ARM64v8') { + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def image = docker.build("${DOCKER_REPO}:${TAG}-arm64", "-f Dockerfile.arm64 .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${DOCKER_REPO}:develop-arm64", "-f Dockerfile.arm64 .") + image.push() + } + } + } + } + } + } + stage('Releases') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + parallel { + stage('Docker Manifest') { + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + sh(script: """ + docker manifest create ${DOCKER_REPO}:${TAG} ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm + docker manifest inspect ${DOCKER_REPO}:${TAG} + docker manifest push -p ${DOCKER_REPO}:${TAG} + docker manifest create ${DOCKER_REPO}:latest ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm + docker manifest inspect ${DOCKER_REPO}:latest + docker manifest push -p ${DOCKER_REPO}:latest + """ + ) + } else if (BRANCH_NAME == 'develop') { + sh(script: """ + docker manifest create ${DOCKER_REPO}:develop ${DOCKER_REPO}:develop-amd64 ${DOCKER_REPO}:develop-arm64 ${DOCKER_REPO}:develop-arm + docker manifest inspect ${DOCKER_REPO}:develop + docker manifest push -p ${DOCKER_REPO}:develop + """ + ) + } + } + } + } + stage('GitHub') { + when { branch 'master' } + agent { label 'amd64'} + steps { + sh """ + git remote set-url origin "https://${GIT_TOKEN_USR}:${GIT_TOKEN_PSW}@github.com/${GIT_REPO}.git" + git tag ${TAG} + git push --tags + """ + } + } + stage('PyPi') { + when { branch 'master' } + agent { label 'amd64'} + steps { + sh """ + python3 -m venv venv && venv/bin/pip install twine + venv/bin/python setup.py sdist && venv/bin/python -m twine --skip-existing -u ${PYPI_CREDS_USR} -p ${PYPI_CREDS_PSW} upload dist/* + git tag ${TAG} + git push --tags + """ + } + } + } + } + } +} diff --git a/README.md b/README.md index b04c69ce..f30f4ef2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Discord](https://img.shields.io/discord/532695326117593112.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/qHNByUW) [![BuyUsCoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://buymeacoff.ee/ouroboros) -[![Travis](https://img.shields.io/travis/pyouroboros/ouroboros/master.svg?style=flat-square)](https://travis-ci.org/pyouroboros/ouroboros) +[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Ouroboros/master)](https://jenkins.cajun.pro/job/Ouroboros/job/master/) [![Release](https://img.shields.io/github/release/pyouroboros/ouroboros.svg?style=flat-square)](https://hub.docker.com/r/pyouroboros/ouroboros/) [![Pypi Downloads](https://img.shields.io/pypi/dm/ouroboros-cli.svg?style=flat-square)](https://pypi.org/project/ouroboros-cli/) [![Python Version](https://img.shields.io/pypi/pyversions/ouroboros-cli.svg?style=flat-square)](https://pypi.org/project/ouroboros-cli/) diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 0f4c7340..00000000 --- a/deploy.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Travis-ci convenience environment vars used: -# TRAVIS_BRANCH | branch name -# $TRAVIS_REPO_SLUG | organization/project (GitHub Capitalization) -# Travis-ci manual environment vars used: -# GITHUB_USER | github username -# GITHUB_TOKEN | $GITHUB_USER's token -# DOCKER_USER | docker username -# DOCKER_PASSWORD | $DOCKER_USER's password - -VERSION="$(grep -i version pyouroboros/__init__.py | cut -d' ' -f3 | tr -d \")" - -# Set branch to latest if master, else keep the same -if [[ "$TRAVIS_BRANCH" == "master" ]]; then - BRANCH="latest" -else - BRANCH="$TRAVIS_BRANCH" -fi - -# get the docker lowercase variant of the repo_name -REPOSITORY="$(echo $TRAVIS_REPO_SLUG | tr '[:upper:]' '[:lower:]')" - -# Docker experimental config -echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json -[[ -d ~/.docker ]] || mkdir ~/.docker -[[ -f ~/.docker/config.json ]] || touch ~/.docker/config.json -echo '{"experimental":"enabled"}' | sudo tee ~/.docker/config.json -sudo service docker restart - -# Auth -echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin - -# Prepare QEMU for ARM builds -docker run --rm --privileged multiarch/qemu-user-static:register --reset -bash prebuild.sh -chmod +x qemu-aarch64-static qemu-arm-static - -# Set tag based off of branch -if [[ "$BRANCH" == "latest" ]]; then - TAG="$VERSION" -else - TAG="$BRANCH" -fi - -# AMDx64 -docker build -t "${REPOSITORY}:${TAG}-amd64" . && \ -docker push "${REPOSITORY}:${TAG}-amd64" - -# Create Initial Manifests -docker manifest create "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-amd64" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest create "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-amd64" -fi - -# ARM variants -for i in $(ls *arm*); do - ARCH="$(echo ${i} | cut -d. -f2)" - docker build -f "Dockerfile.${ARCH}" -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ - docker push "${REPOSITORY}:${TAG}-${ARCH}" - # Add variant to manifest - docker manifest create -a "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" - if [[ "$BRANCH" == "latest" ]]; then - docker manifest create -a "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" - fi -done - -docker manifest inspect "${REPOSITORY}:${TAG}" && \ -docker manifest push "${REPOSITORY}:${TAG}" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest inspect "${REPOSITORY}:${BRANCH}" && \ - docker manifest push "${REPOSITORY}:${BRANCH}" -fi - -# Git tags -if [[ "$BRANCH" == "latest" ]]; then - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${REPOSITORY}.git" && \ - git tag "${VERSION}" && \ - git push --tags -fi \ No newline at end of file diff --git a/prebuild.sh b/prebuild.sh deleted file mode 100644 index 8d12fd72..00000000 --- a/prebuild.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" \ No newline at end of file diff --git a/pyouroboros/__init__.py b/pyouroboros/__init__.py index aaee78f6..5ae08196 100644 --- a/pyouroboros/__init__.py +++ b/pyouroboros/__init__.py @@ -1,2 +1,2 @@ -VERSION = "1.1.2" +VERSION = "1.2.0" BRANCH = "master" diff --git a/pyouroboros/config.py b/pyouroboros/config.py index c5df05ea..1f430807 100644 --- a/pyouroboros/config.py +++ b/pyouroboros/config.py @@ -5,10 +5,10 @@ class Config(object): options = ['INTERVAL', 'PROMETHEUS', 'DOCKER_SOCKETS', 'MONITOR', 'IGNORE', 'LOG_LEVEL', 'PROMETHEUS_ADDR', - 'PROMETHEUS_PORT', 'NOTIFIERS', 'REPO_USER', 'REPO_PASS', 'CLEANUP', 'RUN_ONCE', 'LATEST', 'CRON', + 'PROMETHEUS_PORT', 'NOTIFIERS', 'REPO_USER', 'REPO_PASS', 'CLEANUP', 'RUN_ONCE', 'CRON', 'INFLUX_URL', 'INFLUX_PORT', 'INFLUX_USERNAME', 'INFLUX_PASSWORD', 'INFLUX_DATABASE', 'INFLUX_SSL', 'INFLUX_VERIFY_SSL', 'DATA_EXPORT', 'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS', 'LABELS_ONLY', - 'DRY_RUN', 'HOSTNAME', 'DOCKER_TLS_VERIFY'] + 'DRY_RUN', 'HOSTNAME', 'DOCKER_TLS_VERIFY', 'SWARM'] hostname = environ.get('HOSTNAME') interval = 300 @@ -16,11 +16,11 @@ class Config(object): docker_sockets = 'unix://var/run/docker.sock' docker_tls = False docker_tls_verify = True + swarm = False monitor = [] ignore = [] data_export = None log_level = 'info' - latest = False cleanup = False run_once = False dry_run = False @@ -90,7 +90,7 @@ def parse(self): setattr(self, option.lower(), opt) except ValueError as e: print(e) - elif option in ['LATEST', 'CLEANUP', 'RUN_ONCE', 'INFLUX_SSL', 'INFLUX_VERIFY_SSL', 'DRY_RUN', + elif option in ['CLEANUP', 'RUN_ONCE', 'INFLUX_SSL', 'INFLUX_VERIFY_SSL', 'DRY_RUN', 'SWARM', 'SELF_UPDATE', 'LABEL_ENABLE', 'DOCKER_TLS', 'LABELS_ONLY', 'DOCKER_TLS_VERIFY']: if env_opt.lower() in ['true', 'yes']: setattr(self, option.lower(), True) @@ -111,6 +111,9 @@ def parse(self): if self.interval < 30: self.interval = 30 + if self.labels_only and not self.label_enable: + self.logger.warning('labels_only enabled but not in use without label_enable') + for option in ['docker_sockets', 'notifiers', 'monitor', 'ignore']: if isinstance(getattr(self, option), str): string_list = getattr(self, option) diff --git a/pyouroboros/dockerclient.py b/pyouroboros/dockerclient.py index 56ff4900..9299d824 100644 --- a/pyouroboros/dockerclient.py +++ b/pyouroboros/dockerclient.py @@ -13,9 +13,7 @@ def __init__(self, socket, config, data_manager, notification_manager): self.socket = socket self.client = self.connect() self.data_manager = data_manager - self.data_manager.total_updated[self.socket] = 0 self.logger = getLogger() - self.monitored = self.monitor_filter() self.notification_manager = notification_manager @@ -56,69 +54,85 @@ def connect(self): return client - def get_running(self): - """Return running container objects list, except ouroboros itself""" - running_containers = [] - try: - for container in self.client.containers.list(filters={'status': 'running'}): - if self.config.self_update: - running_containers.append(container) - else: - try: - if 'ouroboros' not in container.image.tags[0]: - running_containers.append(container) - except IndexError: - self.logger.error("%s has no tags.. you should clean it up! Ignoring.", container.id) - continue - except DockerException: - self.logger.critical("Can't connect to Docker API at %s", self.config.docker_socket) - exit(1) +class Container(object): + def __init__(self, docker_client): + self.docker = docker_client + self.logger = self.docker.logger + self.config = self.docker.config + self.client = self.docker.client + self.socket = self.docker.socket + self.data_manager = self.docker.data_manager + self.data_manager.total_updated[self.socket] = 0 + self.notification_manager = self.docker.notification_manager - return running_containers + self.monitored = self.monitor_filter() - def monitor_filter(self): - """Return filtered running container objects list""" - running_containers = self.get_running() - monitored_containers = [] + # Container sub functions + def stop(self, container): + self.logger.debug('Stopping container: %s', container.name) + stop_signal = container.labels.get('com.ouroboros.stop_signal', False) + if stop_signal: + try: + container.kill(signal=stop_signal) + except APIError as e: + self.logger.error('Cannot kill container using signal %s. stopping normally. Error: %s', + stop_signal, e) + container.stop() + else: + container.stop() - for container in running_containers: - ouro_label = container.labels.get('com.ouroboros.enable', False) - # if labels enabled, use the label. 'true/yes' trigger monitoring. - if self.config.label_enable and ouro_label: - if ouro_label.lower() in ["true", "yes"]: - monitored_containers.append(container) - else: - continue - elif not self.config.labels_only and self.config.monitor and container.name in self.config.monitor \ - and container.name not in self.config.ignore: - monitored_containers.append(container) - elif not self.config.labels_only and container.name not in self.config.ignore: - monitored_containers.append(container) + def remove(self, container): + self.logger.debug('Removing container: %s', container.name) + try: + container.remove() + except NotFound as e: + self.logger.error("Could not remove container. Error: %s", e) + return - self.data_manager.monitored_containers[self.socket] = len(monitored_containers) - self.data_manager.set(self.socket) + def recreate(self, container, latest_image): + new_config = set_properties(old=container, new=latest_image) - return monitored_containers + self.stop(container) + self.remove(container) - def pull(self, image_object): - """Docker pull image tag/latest""" - image = image_object - try: - tag = image.tags[0] - except IndexError: - self.logger.error('Malformed or missing tag. Skipping...') - raise ConnectionError - if self.config.latest and image.tags[0][-6:] != 'latest': - if ':' in tag: - split_tag = tag.split(':') - if len(split_tag) == 2: - if '/' not in split_tag[1]: - tag = split_tag[0] + created = self.client.api.create_container(**new_config) + new_container = self.client.containers.get(created.get("Id")) + + # connect the new container to all networks of the old container + for network_name, network_config in container.attrs['NetworkSettings']['Networks'].items(): + network = self.client.networks.get(network_config['NetworkID']) + try: + network.disconnect(new_container.id, force=True) + except APIError: + pass + new_network_config = { + 'container': new_container, + 'aliases': network_config['Aliases'], + 'links': network_config['Links'], + 'ipv4_address': network_config['IPAddress'], + 'ipv6_address': network_config['GlobalIPv6Address'] + } + try: + network.connect(**new_network_config) + except APIError as e: + if any(err in str(e) for err in ['user configured subnets', 'user defined networks']): + if new_network_config.get('ipv4_address'): + del new_network_config['ipv4_address'] + if new_network_config.get('ipv6_address'): + del new_network_config['ipv6_address'] + network.connect(**new_network_config) else: - tag = ':'.join(split_tag[:-1]) - tag = f'{tag}:latest' + self.logger.error('Unable to attach updated container to network "%s". Error: %s', network.name, e) + + new_container.start() + def pull(self, current_tag): + """Docker pull image tag""" + tag = current_tag + if not tag: + self.logger.error('Missing tag. Skipping...') + raise ConnectionError self.logger.debug('Checking tag: %s', tag) try: if self.config.dry_run: @@ -143,20 +157,69 @@ def pull(self, image_object): self.logger.critical("Invalid Credentials. Exiting") exit(1) elif 'Client.Timeout' in str(e): - self.logger.critical("Couldn't find an image on docker.com for %s. Local Build?", image.tags[0]) + self.logger.critical("Couldn't find an image on docker.com for %s. Local Build?", tag) raise ConnectionError elif ('pull access' or 'TLS handshake') in str(e): self.logger.critical("Couldn't pull. Skipping. Error: %s", e) raise ConnectionError - def update_containers(self): - updated_count = 0 - updated_container_tuples = [] - depends_on_list = [] + # Filters + def running_filter(self): + """Return running container objects list, except ouroboros itself""" + running_containers = [] + try: + for container in self.client.containers.list(filters={'status': 'running'}): + if self.config.self_update: + running_containers.append(container) + else: + try: + if 'ouroboros' not in container.image.tags[0]: + running_containers.append(container) + except IndexError: + self.logger.error("%s has no tags.. you should clean it up! Ignoring.", container.id) + continue + + except DockerException: + self.logger.critical("Can't connect to Docker API at %s", self.config.docker_socket) + exit(1) + + return running_containers + + def monitor_filter(self): + """Return filtered running container objects list""" + running_containers = self.running_filter() + monitored_containers = [] + + for container in running_containers: + ouro_label = container.labels.get('com.ouroboros.enable', False) + # if labels enabled, use the label. 'true/yes' trigger monitoring. + if self.config.label_enable and ouro_label: + if ouro_label.lower() in ["true", "yes"]: + monitored_containers.append(container) + else: + continue + elif not self.config.labels_only: + if self.config.monitor: + if container.name in self.config.monitor and container.name not in self.config.ignore: + monitored_containers.append(container) + elif container.name not in self.config.ignore: + monitored_containers.append(container) + + self.data_manager.monitored_containers[self.socket] = len(monitored_containers) + self.data_manager.set(self.socket) + + return monitored_containers + + # Socket Functions + def socket_check(self): + depends_on_names = [] + hard_depends_on_names = [] + updateable = [] self.monitored = self.monitor_filter() if not self.monitored: self.logger.info('No containers are running or monitored on %s', self.socket) + return me_list = [c for c in self.client.api.containers() if 'ouroboros' in c['Names'][0].strip('/')] if len(me_list) > 1: @@ -164,16 +227,59 @@ def update_containers(self): for container in self.monitored: current_image = container.image - - shared_image = [uct for uct in updated_container_tuples if uct[1].id == current_image.id] + current_tag = container.attrs['Config']['Image'] + shared_image = [uct for uct in updateable if uct[1].id == current_image.id] if shared_image: latest_image = shared_image[0][2] else: try: - latest_image = self.pull(current_image) + latest_image = self.pull(current_tag) except ConnectionError: continue + if current_image.id != latest_image.id: + updateable.append((container, current_image, latest_image)) + else: + continue + + # Get container list to restart after update complete + depends_on = container.labels.get('com.ouroboros.depends_on', False) + hard_depends_on = container.labels.get('com.ouroboros.hard_depends_on', False) + if depends_on: + depends_on_names.extend([name.strip() for name in depends_on.split(',')]) + if hard_depends_on: + hard_depends_on_names.extend([name.strip() for name in hard_depends_on.split(',')]) + + hard_depends_on_containers = [] + hard_depends_on_names = list(set(hard_depends_on_names)) + for name in hard_depends_on_names: + try: + hard_depends_on_containers.append(self.client.containers.get(name)) + except NotFound: + self.logger.error("Could not find dependant container %s on socket %s. Ignoring", name, self.socket) + + depends_on_containers = [] + depends_on_names = list(set(depends_on_names)) + depends_on_names = [name for name in depends_on_names if name not in hard_depends_on_names] + for name in depends_on_names: + try: + depends_on_containers.append(self.client.containers.get(name)) + except NotFound: + self.logger.error("Could not find dependant container %s on socket %s. Ignoring", name, self.socket) + + return updateable, depends_on_containers, hard_depends_on_containers + + def update(self): + updated_count = 0 + try: + updateable, depends_on_containers, hard_depends_on_containers = self.socket_check() + except TypeError: + return + + for container in depends_on_containers + hard_depends_on_containers: + self.stop(container) + + for container, current_image, latest_image in updateable: if self.config.dry_run: # Ugly hack for repo digest repo_digest_id = current_image.attrs['RepoDigests'][0].split('@')[1] @@ -181,79 +287,41 @@ def update_containers(self): self.logger.info('dry run : %s would be updated', container.name) continue - # If current running container is running latest image - if current_image.id != latest_image.id: - updated_container_tuples.append( - (container, current_image, latest_image) - ) - - if container.name in ['ouroboros', 'ouroboros-updated']: - self.data_manager.total_updated[self.socket] += 1 - self.data_manager.add(label=container.name, socket=self.socket) - self.data_manager.add(label='all', socket=self.socket) - self.notification_manager.send(container_tuples=updated_container_tuples, - socket=self.socket, kind='update') - self.update_self(old_container=container, new_image=latest_image, count=1) - - self.logger.info('%s will be updated', container.name) - - # Get container list to restart after update complete - depends_on = container.labels.get('com.ouroboros.depends-on', False) - if depends_on: - depends_on_list.extend([name.strip() for name in depends_on.split(',')]) - # new container dict to create new container from - new_config = set_properties(old=container, new=latest_image) - - self.logger.debug('Stopping container: %s', container.name) - stop_signal = container.labels.get('com.ouroboros.stop-signal', False) - if stop_signal: - try: - container.kill(signal=stop_signal) - except APIError as e: - self.logger.error('Cannot kill container using signal %s. stopping normally. Error: %s', - stop_signal, e) - container.stop() - else: - container.stop() + if container.name in ['ouroboros', 'ouroboros-updated']: + self.data_manager.total_updated[self.socket] += 1 + self.data_manager.add(label=container.name, socket=self.socket) + self.data_manager.add(label='all', socket=self.socket) + self.notification_manager.send(container_tuples=updateable, + socket=self.socket, kind='update') + self.update_self(old_container=container, new_image=latest_image, count=1) - self.logger.debug('Removing container: %s', container.name) - try: - container.remove() - except NotFound as e: - self.logger.error("Could not remove container. Error: %s", e) + self.logger.info('%s will be updated', container.name) - created = self.client.api.create_container(**new_config) - new_container = self.client.containers.get(created.get("Id")) - new_container.start() + self.recreate(container, latest_image) - if self.config.cleanup: - try: - self.client.images.remove(current_image.id) - except APIError as e: - self.logger.error("Could not delete old image for %s, Error: %s", container.name, e) - updated_count += 1 + if self.config.cleanup: + try: + self.client.images.remove(current_image.id) + except APIError as e: + self.logger.error("Could not delete old image for %s, Error: %s", container.name, e) + updated_count += 1 - self.logger.debug("Incrementing total container updated count") + self.logger.debug("Incrementing total container updated count") - self.data_manager.total_updated[self.socket] += 1 - self.data_manager.add(label=container.name, socket=self.socket) - self.data_manager.add(label='all', socket=self.socket) + self.data_manager.total_updated[self.socket] += 1 + self.data_manager.add(label=container.name, socket=self.socket) + self.data_manager.add(label='all', socket=self.socket) - if depends_on_list: - depends_on_containers = [] - for name in list(set(depends_on_list)): - try: - depends_on_containers.append(self.client.containers.get(name)) - except NotFound: - self.logger.error("Could not find dependant container %s on socket %s. Ignoring", name, self.socket) + for container in depends_on_containers: + # Reload container to ensure it isn't referencing the old image + container.reload() + container.start() - if depends_on_containers: - for container in depends_on_containers: - self.logger.debug('Restarting dependant container %s', container.name) - container.restart() + for container in hard_depends_on_containers: + self.recreate(container, container.image) if updated_count > 0: - self.notification_manager.send(container_tuples=updated_container_tuples, socket=self.socket, kind='update') + self.notification_manager.send(container_tuples=updateable, socket=self.socket, kind='update') def update_self(self, count=None, old_container=None, me_list=None, new_image=None): if count == 2: @@ -279,3 +347,124 @@ def update_self(self, count=None, old_container=None, me_list=None, new_image=No self.logger.debug('If you strike me down, I shall become more powerful than you could possibly imagine') self.logger.debug('https://bit.ly/2VVY7GH') sleep(30) + + +class Service(object): + def __init__(self, docker_client): + self.docker = docker_client + self.logger = self.docker.logger + self.config = self.docker.config + self.client = self.docker.client + self.socket = self.docker.socket + self.data_manager = self.docker.data_manager + self.data_manager.total_updated[self.socket] = 0 + self.notification_manager = self.docker.notification_manager + + self.monitored = self.monitor_filter() + + def monitor_filter(self): + """Return filtered service objects list""" + services = self.client.services.list(filters={'label': 'com.ouroboros.enable'}) + + monitored_services = [] + + for service in services: + ouro_label = service.attrs['Spec']['Labels'].get('com.ouroboros.enable') + if ouro_label.lower() in ["true", "yes"]: + monitored_services.append(service) + + self.data_manager.monitored_containers[self.socket] = len(monitored_services) + self.data_manager.set(self.socket) + + return monitored_services + + def pull(self, tag): + """Docker pull image tag""" + self.logger.debug('Checking tag: %s', tag) + try: + if self.config.dry_run: + registry_data = self.client.images.get_registry_data(tag) + return registry_data + else: + if self.config.auth_json: + return_image = self.client.images.pull(tag, auth_config=self.config.auth_json) + else: + return_image = self.client.images.pull(tag) + return return_image + except APIError as e: + if '' in str(e): + self.logger.debug("Docker api issue. Ignoring") + raise ConnectionError + elif 'unauthorized' in str(e): + if self.config.dry_run: + self.logger.error('dry run : Upstream authentication issue while checking %s. See: ' + 'https://github.com/docker/docker-py/issues/2225', tag) + raise ConnectionError + else: + self.logger.critical("Invalid Credentials. Exiting") + exit(1) + elif 'Client.Timeout' in str(e): + self.logger.critical("Couldn't find an image on docker.com for %s. Local Build?", tag) + raise ConnectionError + elif ('pull access' or 'TLS handshake') in str(e): + self.logger.critical("Couldn't pull. Skipping. Error: %s", e) + raise ConnectionError + + def update(self): + updated_count = 0 + updated_service_tuples = [] + self.monitored = self.monitor_filter() + + if not self.monitored: + self.logger.info('No services monitored') + + for service in self.monitored: + image_string = service.attrs['Spec']['TaskTemplate']['ContainerSpec']['Image'] + if '@' in image_string: + tag = image_string.split('@')[0] + sha256 = image_string.split('@')[1][7:] + else: + self.logger.error('No image SHA for %s. Skipping', image_string) + continue + + try: + latest_image = self.pull(tag) + except ConnectionError: + continue + + if self.config.dry_run: + # Ugly hack for repo digest + if sha256 != latest_image.id: + self.logger.info('dry run : %s would be updated', service.name) + continue + + if sha256 != latest_image.id: + updated_service_tuples.append( + (service, sha256[-10:], latest_image) + ) + + if 'ouroboros' in service.name and self.config.self_update: + self.data_manager.total_updated[self.socket] += 1 + self.data_manager.add(label=service.name, socket=self.socket) + self.data_manager.add(label='all', socket=self.socket) + self.notification_manager.send(container_tuples=updated_service_tuples, + socket=self.socket, kind='update', mode='service') + + self.logger.info('%s will be updated', service.name) + service.update(image=tag) + + updated_count += 1 + + self.logger.debug("Incrementing total service updated count") + + self.data_manager.total_updated[self.socket] += 1 + self.data_manager.add(label=service.name, socket=self.socket) + self.data_manager.add(label='all', socket=self.socket) + + if updated_count > 0: + self.notification_manager.send( + container_tuples=updated_service_tuples, + socket=self.socket, + kind='update', + mode='service' + ) diff --git a/pyouroboros/logger.py b/pyouroboros/logger.py index 9d0c4ff3..abcdfaab 100644 --- a/pyouroboros/logger.py +++ b/pyouroboros/logger.py @@ -47,3 +47,7 @@ def __init__(self, level='INFO'): # Add the Handler to the Logger self.logger.addHandler(console_logger) + + # Less verbose apscheduler logging if info + if self.logger.getEffectiveLevel() == 20: + getLogger('apscheduler').setLevel('WARNING') diff --git a/pyouroboros/notifiers.py b/pyouroboros/notifiers.py index bc2a3e35..c93a4ac7 100644 --- a/pyouroboros/notifiers.py +++ b/pyouroboros/notifiers.py @@ -32,7 +32,7 @@ def build_apprise(self): return apprise_obj - def send(self, container_tuples=None, socket=None, kind='update', next_run=None): + def send(self, container_tuples=None, socket=None, kind='update', next_run=None, mode='container'): if kind == 'startup': now = datetime.now(timezone.utc).astimezone() title = f'Ouroboros has started' @@ -52,7 +52,7 @@ def send(self, container_tuples=None, socket=None, kind='update', next_run=None) [ "{} updated from {} to {}".format( container.name, - old_image.short_id.split(':')[1], + old_image if mode == 'service' else old_image.short_id.split(':')[1], new_image.short_id.split(':')[1] ) for container, old_image, new_image in container_tuples ] diff --git a/pyouroboros/ouroboros.py b/pyouroboros/ouroboros.py index 58ca7f3a..5223edc5 100644 --- a/pyouroboros/ouroboros.py +++ b/pyouroboros/ouroboros.py @@ -8,10 +8,10 @@ from pyouroboros.config import Config from pyouroboros import VERSION, BRANCH -from pyouroboros.dockerclient import Docker from pyouroboros.logger import OuroborosLogger from pyouroboros.dataexporters import DataManager from pyouroboros.notifiers import NotificationManager +from pyouroboros.dockerclient import Docker, Container, Service def main(): @@ -50,6 +50,9 @@ def main(): core_group.add_argument('-u', '--self-update', default=Config.self_update, dest='SELF_UPDATE', action='store_true', help='Let ouroboros update itself') + core_group.add_argument('-S', '--swarm', default=Config.swarm, dest='SWARM', action='store_true', + help='Put ouroboros in swarm mode') + core_group.add_argument('-o', '--run-once', default=Config.run_once, action='store_true', dest='RUN_ONCE', help='Single run') @@ -83,9 +86,6 @@ def main(): docker_group.add_argument('-c', '--cleanup', default=Config.cleanup, dest='CLEANUP', action='store_true', help='Remove old images after updating') - docker_group.add_argument('-L', '--latest', default=Config.latest, dest='LATEST', action='store_true', - help='Check for latest image instead of pulling current tag') - docker_group.add_argument('-r', '--repo-user', default=Config.repo_user, dest='REPO_USER', help='Private docker registry username\n' 'EXAMPLE: foo@bar.baz') @@ -151,9 +151,13 @@ def main(): for socket in config.docker_sockets: try: docker = Docker(socket, config, data_manager, notification_manager) + if config.swarm: + mode = Service(docker) + else: + mode = Container(docker) if config.cron: scheduler.add_job( - docker.update_containers, + mode.update, name=f'Cron container update for {socket}', trigger='cron', minute=config.cron[0], @@ -164,14 +168,14 @@ def main(): ) else: if config.run_once: - scheduler.add_job(docker.update_containers, name=f'Run Once container update for {socket}') + scheduler.add_job(mode.update, name=f'Run Once container update for {socket}') else: scheduler.add_job( - docker.update_containers, + mode.update, name=f'Initial run interval container update for {socket}' ) scheduler.add_job( - docker.update_containers, + mode.update, name=f'Interval container update for {socket}', trigger='interval', seconds=config.interval )