From 1ec56e2eff5adc3d3938f0862b14591518ca51d7 Mon Sep 17 00:00:00 2001 From: Patrick Valsecchi Date: Thu, 8 Jun 2017 13:47:13 +0200 Subject: [PATCH] Add Alembic health checks that validates the DB is at the last version --- .gitignore | 6 +- .idea/sqldialects.xml | 6 ++ README.md | 8 +- acceptance_tests/app/alembic.ini | 32 ++++++++ acceptance_tests/app/app_alembic/__init__.py | 0 acceptance_tests/app/app_alembic/env.py | 76 +++++++++++++++++++ .../app/app_alembic/script.py.mako | 24 ++++++ .../versions/4a8c1bb4e775_initial_version.py | 30 ++++++++ .../app/app_alembic/versions/__init__.py | 0 .../app/c2cwsgiutils_app/__init__.py | 1 + acceptance_tests/app/requirements.txt | 1 + acceptance_tests/app/run_alembic.sh | 19 +++++ acceptance_tests/tests/docker-compose.yml | 18 ++++- acceptance_tests/tests/tests/conftest.py | 15 ++-- .../tests/tests/test_health_check.py | 6 +- c2cwsgiutils/db.py | 4 +- c2cwsgiutils/health_check.py | 73 ++++++++++++++++-- c2cwsgiutils/pyramid.py | 1 + c2cwsgiutils/stats.py | 5 ++ c2cwsgiutils/stats_pyramid.py | 2 + c2cwsgiutils/wsgi.py | 1 + setup.py | 2 +- 22 files changed, 303 insertions(+), 27 deletions(-) create mode 100644 .idea/sqldialects.xml create mode 100644 acceptance_tests/app/alembic.ini create mode 100644 acceptance_tests/app/app_alembic/__init__.py create mode 100644 acceptance_tests/app/app_alembic/env.py create mode 100644 acceptance_tests/app/app_alembic/script.py.mako create mode 100644 acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py create mode 100644 acceptance_tests/app/app_alembic/versions/__init__.py create mode 100755 acceptance_tests/app/run_alembic.sh diff --git a/.gitignore b/.gitignore index bb2ffae93..699d94d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +__pycache__ +*.pyc /.venv/ /acceptance_tests/app/c2cwsgiutils /acceptance_tests/app/c2cwsgiutils_coverage_report.py @@ -13,5 +15,5 @@ /c2cwsgiutils.egg-info/ /dist/ /reports/ -.idea/dbnavigator.xml -.idea/inspectionProfiles +/.idea/dbnavigator.xml +/.idea/inspectionProfiles diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 000000000..6df4889b0 --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index bd32526d8..a96e298cc 100644 --- a/README.md +++ b/README.md @@ -182,15 +182,17 @@ health_check = HealthCheck(config) health_check.add_db_session_check(models.DBSession, at_least_one_model=models.Hello) health_check.add_url_check('http://localhost/api/hello') health_check.add_custom_check('custom', custom_check, 2) +health_check.add_alembic_check(models.DBSession, '/app/alembic.ini', 3) ``` -Then, the URL `{C2C_BASE_PATH}/health_check?max_level=2` can be used to run the health checks and get a report +Then, the URL `{C2C_BASE_PATH}/health_check?max_level=3` can be used to run the health checks and get a report looking like that (in case of error): ```json { "status": 500, - "successes": ["db_engine_sqlalchemy", "db_engine_sqlalchemy_slave", "http://localhost/api/hello"], + "successes": ["db_engine_sqlalchemy", "db_engine_sqlalchemy_slave", "http://localhost/api/hello", + "alembic_app_alembic.ini"], "failures": { "custom": { "message": "I'm not happy" @@ -199,6 +201,8 @@ looking like that (in case of error): } ``` +Look at the documentation of the `c2cwsgiutils.health_check.HealthCheck` class for more information. + SQLAlchemy models graph ----------------------- diff --git a/acceptance_tests/app/alembic.ini b/acceptance_tests/app/alembic.ini new file mode 100644 index 000000000..956d1ab51 --- /dev/null +++ b/acceptance_tests/app/alembic.ini @@ -0,0 +1,32 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = app_alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = %(SQLALCHEMY_URL)s diff --git a/acceptance_tests/app/app_alembic/__init__.py b/acceptance_tests/app/app_alembic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance_tests/app/app_alembic/env.py b/acceptance_tests/app/app_alembic/env.py new file mode 100644 index 000000000..510dc8273 --- /dev/null +++ b/acceptance_tests/app/app_alembic/env.py @@ -0,0 +1,76 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import os + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +config.set_main_option('sqlalchemy.url', os.environ["SQLALCHEMY_URL"]) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig('production.ini', defaults=os.environ) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + # To allow to run "UPDATE" in a migration and do an "ALTER" in another one, later. + transaction_per_migration=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/acceptance_tests/app/app_alembic/script.py.mako b/acceptance_tests/app/app_alembic/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/acceptance_tests/app/app_alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py b/acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py new file mode 100644 index 000000000..fca0e4f65 --- /dev/null +++ b/acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py @@ -0,0 +1,30 @@ +"""Initial version + +Revision ID: 4a8c1bb4e775 +Revises: +Create Date: 2016-09-14 09:23:27.466418 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4a8c1bb4e775' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS postgis;") + op.execute(""" + CREATE TABLE hello ( + id SERIAL PRIMARY KEY, + value TEXT UNIQUE INITIALLY DEFERRED + ) + """) + + +def downgrade(): + pass diff --git a/acceptance_tests/app/app_alembic/versions/__init__.py b/acceptance_tests/app/app_alembic/versions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance_tests/app/c2cwsgiutils_app/__init__.py b/acceptance_tests/app/c2cwsgiutils_app/__init__.py index bb6812e64..30c53cf4f 100644 --- a/acceptance_tests/app/c2cwsgiutils_app/__init__.py +++ b/acceptance_tests/app/c2cwsgiutils_app/__init__.py @@ -22,5 +22,6 @@ def main(_, **settings): health_check.add_db_session_check(models.DBSession, at_least_one_model=models.Hello) health_check.add_url_check('http://localhost/api/hello') health_check.add_custom_check('fail', _failure, 2) + health_check.add_alembic_check(models.DBSession, '/app/alembic.ini', 1) return config.make_wsgi_app() diff --git a/acceptance_tests/app/requirements.txt b/acceptance_tests/app/requirements.txt index da4819bf9..33e4fde22 100644 --- a/acceptance_tests/app/requirements.txt +++ b/acceptance_tests/app/requirements.txt @@ -1,2 +1,3 @@ -r rel_requirements.txt +alembic==0.9.2 flake8==3.3.0 diff --git a/acceptance_tests/app/run_alembic.sh b/acceptance_tests/app/run_alembic.sh new file mode 100755 index 000000000..80e10a844 --- /dev/null +++ b/acceptance_tests/app/run_alembic.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Upgrade the DB +set -e + +# wait for the DB to be UP +while ! echo "import sqlalchemy; sqlalchemy.create_engine('$SQLALCHEMY_URL').connect()" | python 2> /dev/null +do + echo "Waiting for the DB to be reachable" + sleep 1; +done + +for ini in *alembic*.ini +do + if [[ -f $ini ]] + then + echo "$ini ===========================" + alembic -c $ini upgrade head + fi +done diff --git a/acceptance_tests/tests/docker-compose.yml b/acceptance_tests/tests/docker-compose.yml index 4d45abdad..1e6b7b47a 100644 --- a/acceptance_tests/tests/docker-compose.yml +++ b/acceptance_tests/tests/docker-compose.yml @@ -10,7 +10,7 @@ app: SQL_PROFILER_SECRET: changeme DEBUG_VIEW_SECRET: changeme LOG_HOST: 172.17.0.1 - LOG_TYPE: 'json,logstash' + LOG_TYPE: 'console,logstash' SQL_LOG_LEVEL: DEBUG OTHER_LOG_LEVEL: DEBUG DEVELOPMENT: 1 @@ -21,6 +21,22 @@ app: ports: - 8480:80 +alembic_master: + image: camptocamp/c2cwsgiutils_test_app:${DOCKER_TAG} + environment: + SQLALCHEMY_URL: postgresql://www-data:www-data@db:5432/test + links: + - db + command: /bin/true # will use execute with another script from the tests to actually do it + +alembic_slave: + image: camptocamp/c2cwsgiutils_test_app:${DOCKER_TAG} + environment: + SQLALCHEMY_URL: postgresql://www-data:www-data@db:5432/test + links: + - db_slave:db + command: /bin/true # will use execute with another script from the tests to actually do it + db: image: camptocamp/postgresql:pg9.5-latest environment: diff --git a/acceptance_tests/tests/tests/conftest.py b/acceptance_tests/tests/tests/conftest.py index 0666883ae..0df8e326e 100644 --- a/acceptance_tests/tests/tests/conftest.py +++ b/acceptance_tests/tests/tests/conftest.py @@ -32,25 +32,20 @@ def app_connection(composition): @pytest.fixture(scope="session") def master_db_setup(composition): - return _create_table(master=True) + return _create_table(composition, master=True) @pytest.fixture(scope="session") def slave_db_setup(composition): - return _create_table(master=False) + return _create_table(composition, master=False) -def _create_table(master): +def _create_table(composition, master): name = 'master' if master else 'slave' + composition.run('alembic_' + name, '/app/run_alembic.sh') connection = _connect(master) with connection.cursor() as curs: - LOG.info("Creating table for " + name) - curs.execute(""" - CREATE TABLE hello ( - id SERIAL PRIMARY KEY, - value TEXT UNIQUE INITIALLY DEFERRED - ) - """) + LOG.info("Creating data for " + name) curs.execute("INSERT INTO hello (value) VALUES ('%s')" % (name)) connection.commit() return connection diff --git a/acceptance_tests/tests/tests/test_health_check.py b/acceptance_tests/tests/tests/test_health_check.py index c73bdf9d9..112d7d888 100644 --- a/acceptance_tests/tests/tests/test_health_check.py +++ b/acceptance_tests/tests/tests/test_health_check.py @@ -6,7 +6,8 @@ def test_ok(app_connection): print('response=' + json.dumps(response)) assert response == { 'status': 200, - 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'], + 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello', + 'alembic_app_alembic.ini'], 'failures': {} } @@ -16,7 +17,8 @@ def test_failure(app_connection): print('response=' + json.dumps(response)) assert response == { 'status': 500, - 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'], + 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello', + 'alembic_app_alembic.ini'], 'failures': { 'fail': { 'message': 'failing check', diff --git a/c2cwsgiutils/db.py b/c2cwsgiutils/db.py index afc6264d4..257a87a31 100644 --- a/c2cwsgiutils/db.py +++ b/c2cwsgiutils/db.py @@ -27,9 +27,9 @@ def setup_session(config, master_prefix, slave_prefix=None, force_master=None, f path includes the route_prefix. :param config: The pyramid Configuration object - :param master_prefix: The prefix for the master connection configuration entries in the application + :param master_prefix: The prefix for the master connection configuration entries in the application \ settings - :param slave_prefix: The prefix for the slave connection configuration entries in the application + :param slave_prefix: The prefix for the slave connection configuration entries in the application \ settings :param force_master: The method/paths that needs to use the master :param force_slave: The method/paths that needs to use the slave diff --git a/c2cwsgiutils/health_check.py b/c2cwsgiutils/health_check.py index 43c423cd5..603143f11 100644 --- a/c2cwsgiutils/health_check.py +++ b/c2cwsgiutils/health_check.py @@ -4,15 +4,40 @@ To use it, create an instance of this class in your application initialization and do a few calls to its methods add_db_check() """ +import configparser import logging import os +from pyramid.httpexceptions import HTTPNotFound +import re +import requests +import subprocess import traceback -import requests from c2cwsgiutils import stats, _utils -from pyramid.httpexceptions import HTTPNotFound LOG = logging.getLogger(__name__) +ALEMBIC_HEAD_RE = re.compile(r'^([a-f0-9]+) \(head\)\n$') + + +def _get_bindings(session): + return [session.c2c_rw_bind, session.c2c_ro_bind] if session.c2c_rw_bind != session.c2c_ro_bind\ + else [session.c2c_rw_bind] + + +def _get_alembic_version(alembic_ini_path): + # Go to the directory holding the config file and add '.' to the PYTHONPATH variable to support Alembic + # migration scripts using common modules + env = dict(os.environ) + pythonpath = os.environ.get('PYTHONPATH', '') + pythonpath = (pythonpath + ':' if pythonpath else '') + '.' + env['PYTHONPATH'] = pythonpath + + out = subprocess.check_output(['alembic', '-c', alembic_ini_path, 'heads'], + cwd=os.path.dirname(alembic_ini_path), env=env).decode('utf-8') + out_match = ALEMBIC_HEAD_RE.match(out) + if not out_match: + raise Exception("Cannot get the alembic HEAD version from: " + out) + return out_match.group(1) class HealthCheck: @@ -25,6 +50,7 @@ def __init__(self, config): def add_db_session_check(self, session, query_cb=None, at_least_one_model=None, level=1): """ Check a DB session is working. You can specify either query_cb or at_least_one_model. + :param session: a DB session created by c2cwsgiutils.db.setup_session() :param query_cb: a callable that take a session as parameter and check it works :param at_least_one_model: a model that must have at least one entry in the DB @@ -32,17 +58,49 @@ def add_db_session_check(self, session, query_cb=None, at_least_one_model=None, """ if query_cb is None: query_cb = self._at_least_one(at_least_one_model) - self._checks.append(self._create_db_engine_check(session, session.c2c_rw_bind, query_cb) + (level,)) - if session.c2c_rw_bind != session.c2c_ro_bind: - self._checks.append(self._create_db_engine_check(session, session.c2c_ro_bind, - query_cb) + (level,)) + for binding in _get_bindings(session): + self._checks.append(self._create_db_engine_check(session, binding, query_cb) + (level,)) + + def add_alembic_check(self, session, alembic_ini_path, level=2): + """ + Check the DB version against the HEAD version of Alembic. + + :param session: A DB session created by c2cwsgiutils.db.setup_session() giving access to the DB \ + managed by Alembic + :param alembic_ini_path: Path to the Alembic INI file. + :param level: the level of the health check + """ + def check(_request): + for binding in _get_bindings(session): + prev_bind = session.bind + try: + session.bind = binding + with stats.timer_context(['sql', 'manual', 'health_check', 'alembic', alembic_ini_path, + binding.c2c_name]): + actual_version, = session.execute( + "SELECT version_num FROM {schema}.{table}".format(schema=schema, table=table) + ).fetchone() + if actual_version != version: + raise Exception("Invalid alembic version: %s != %s" % (actual_version, version)) + finally: + session.bind = prev_bind + + config = configparser.ConfigParser() + config.read(alembic_ini_path) + schema = config['alembic'].get('version_table_schema', 'public') + table = config['alembic'].get('version_table', 'alembic_version') + + version = _get_alembic_version(alembic_ini_path) + + self._checks.append(('alembic_' + alembic_ini_path.replace('/', '_').strip('_'), check, level)) def add_url_check(self, url, name=None, check_cb=lambda request, response: None, timeout=3, level=1): """ Check that a GET on an URL returns 2xx + :param url: the URL to query :param name: the name of the check (defaults to url) - :param check_cb: an optional CB to do additional checks on the response (takes the request and the + :param check_cb: an optional CB to do additional checks on the response (takes the request and the \ response as parameters) :param timeout: the timeout :param level: the level of the health check @@ -58,6 +116,7 @@ def check(request): def add_custom_check(self, name, check_cb, level=1): """ Add a custom check + :param name: the name of the check :param check_cb: the callback to call (takes the request as parameter) :param level: the level of the health check diff --git a/c2cwsgiutils/pyramid.py b/c2cwsgiutils/pyramid.py index 25f29b665..20da0e0f9 100644 --- a/c2cwsgiutils/pyramid.py +++ b/c2cwsgiutils/pyramid.py @@ -7,6 +7,7 @@ def includeme(config): """ Setup all the pyramid services and event handlers provided by this library. + :param config: The pyramid Configuration """ config.add_settings(handle_exceptions=False) diff --git a/c2cwsgiutils/stats.py b/c2cwsgiutils/stats.py index c377b7d6e..5a4b7c8ee 100644 --- a/c2cwsgiutils/stats.py +++ b/c2cwsgiutils/stats.py @@ -19,6 +19,7 @@ def timer_context(key): """ Add a duration measurement to the stats using the duration the context took to run + :param key: The path of the key, given as a list. """ measure = timer(key) @@ -30,6 +31,7 @@ def timer(key=None): """ Create a timer for the given key. The key can be omitted, but then need to be specified when stop is called. + :param key: The path of the key, given as a list. :return: An instance of _Timer """ @@ -40,6 +42,7 @@ def timer(key=None): def set_gauge(key, value): """ Set a gauge value + :param key: The path of the key, given as a list. :param value: The new value of the gauge """ @@ -50,6 +53,7 @@ def set_gauge(key, value): def increment_counter(key, increment=1): """ Increment a counter value + :param key: The path of the key, given as a list. :param increment: The increment """ @@ -172,6 +176,7 @@ def counter(self, key, increment): def init_backends(settings): """ Initialize the backends according to the configuration. + :param config: The Pyramid config """ if _utils.env_or_settings(settings, "STATS_VIEW", "c2c.stats_view", False): # pragma: nocover diff --git a/c2cwsgiutils/stats_pyramid.py b/c2cwsgiutils/stats_pyramid.py index 3cd532b85..c6c97cf40 100644 --- a/c2cwsgiutils/stats_pyramid.py +++ b/c2cwsgiutils/stats_pyramid.py @@ -90,6 +90,7 @@ def init_db_spy(): # pragma: nocover def init_pyramid_spy(config): # pragma: nocover """ Subscribe to Pyramid events in order to get some stats on route time execution. + :param config: The Pyramid config """ config.add_subscriber(_request_callback, pyramid.events.NewRequest) @@ -99,6 +100,7 @@ def init_pyramid_spy(config): # pragma: nocover def init(config): """ Initialize the whole stats module. + :param config: The Pyramid config """ stats.init_backends(config.get_settings()) diff --git a/c2cwsgiutils/wsgi.py b/c2cwsgiutils/wsgi.py index 2c4a678e5..e6078533c 100644 --- a/c2cwsgiutils/wsgi.py +++ b/c2cwsgiutils/wsgi.py @@ -8,6 +8,7 @@ def create_application(configfile=None): """ Create a standard WSGI application with the capabilities to use environment variables in the configuration file (use %(ENV_VAR)s place holders) + :param config: The configuration file to use :return: The application """ diff --git a/setup.py b/setup.py index 35c8b346f..45583f743 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -VERSION = '0.14.2' +VERSION = '0.15.0' HERE = os.path.abspath(os.path.dirname(__file__)) INSTALL_REQUIRES = open(os.path.join(HERE, 'rel_requirements.txt')).read().splitlines()