Add c2cwsgiutils_stats_db.py

A utility to generate statistics (gauges) about the row counts.

.gitignore

@@ -3,6 +3,7 @@
 /acceptance_tests/app/c2cwsgiutils_coverage_report.py
 /acceptance_tests/app/c2cwsgiutils_run
 /acceptance_tests/app/c2cwsgiutils_genversion.py
+/acceptance_tests/app/c2cwsgiutils_stats_db.py
 /acceptance_tests/app/rel_requirements.txt
 /acceptance_tests/app/setup.cfg
 /acceptance_tests/tests/c2cwsgiutils

Jenkinsfile

@@ -7,16 +7,14 @@ env.DOCKER_TAG = env.BUILD_TAG
 
 // We don't want to publish the same branch twice at the same time.
 dockerBuild {
+    checkout scm
     stage('Update docker') {
-        checkout scm
         sh 'make pull'
     }
     stage('Build') {
-        checkout scm
         sh 'make -j2 build'
     }
     stage('Test') {
-        checkout scm
         try {
             lock("acceptance-${env.NODE_NAME}") {  //only one acceptance test at a time on a machine
                 sh 'make -j2 acceptance'

Makefile

@@ -59,7 +59,7 @@ build_acceptance:
 
 .PHONY: build_test_app
 build_test_app:
-	rsync -a c2cwsgiutils c2cwsgiutils_run c2cwsgiutils_genversion.py c2cwsgiutils_coverage_report.py rel_requirements.txt setup.cfg acceptance_tests/app/
+	rsync -a c2cwsgiutils c2cwsgiutils_run c2cwsgiutils_genversion.py c2cwsgiutils_coverage_report.py c2cwsgiutils_stats_db.py rel_requirements.txt setup.cfg acceptance_tests/app/
 	docker build -t $(DOCKER_BASE)_test_app:$(DOCKER_TAG) --build-arg "GIT_TAG=$(GIT_TAG)" --build-arg "GIT_HASH=$(GIT_HASH)" acceptance_tests/app
 
 .venv/timestamp: rel_requirements.txt dev_requirements.txt

README.md

@@ -111,6 +111,9 @@ with stats.timer_context('toto', 'tutu'):
 
 Other functions exists to generate metrics. Look at the `c2cwsgiutils.stats` module.
 
+Look at the `c2cwsgiutils_stats_db.py` utility if you want to generate statistics (gauges) about the
+row counts.
+
 
 SQL profiler
 ------------

acceptance_tests/tests/docker-compose.yml

@@ -43,3 +43,5 @@ db_slave:
 run_test:
   image: camptocamp/c2cwsgiutils_test_app:${DOCKER_TAG}
   command: 'true'
+  links:
+    - db

acceptance_tests/tests/tests/test_stats_db.py

@@ -0,0 +1,20 @@
+import pytest
+import subprocess
+
+
+def test_no_extra(app_connection, composition):
+    composition.run('run_test', './c2cwsgiutils_stats_db.py',
+                    '--db', 'postgresql://www-data:www-data@db:5432/test', '--schema', 'public')
+
+
+def test_with_extra(app_connection, composition):
+    composition.run('run_test', './c2cwsgiutils_stats_db.py',
+                    '--db', 'postgresql://www-data:www-data@db:5432/test', '--schema', 'public',
+                    '--extra', "select 'toto', 42")
+
+
+def test_error(app_connection, composition):
+    with pytest.raises(subprocess.CalledProcessError):
+        composition.run('run_test', './c2cwsgiutils_stats_db.py',
+                        '--db', 'postgresql://www-data:www-data@db:5432/test', '--schema', 'public',
+                        '--extra', "select 'toto, 42")

c2cwsgiutils/_auth.py

@@ -0,0 +1,9 @@
+from pyramid.httpexceptions import HTTPForbidden
+
+from c2cwsgiutils._utils import env_or_settings
+
+
+def auth_view(request, env_name, config_name):
+    if request.params.get('secret') != env_or_settings(request.registry.settings, env_name, config_name,
+                                                       False):
+        raise HTTPForbidden('Missing or invalid secret parameter')

c2cwsgiutils/_utils.py

@@ -1,21 +1,18 @@
 """
 Private utilities.
 """
-
 import os
-from pyramid.httpexceptions import HTTPForbidden
 
 
 def get_base_path(config):
     return env_or_config(config, 'C2C_BASE_PATH', 'c2c.base_path', '')
 
 
 def env_or_config(config, env_name, config_name, default):
-    if env_name in os.environ:
-        return os.environ[env_name]
-    return config.get_settings().get(config_name, default)
+    return env_or_settings(config.get_settings(), env_name, config_name, default)
 
 
-def auth_view(request, env_name, config_name):
-    if request.params.get('secret') != env_or_config(request.registry.settings, env_name, config_name, False):
-        raise HTTPForbidden('Missing or invalid secret parameter')
+def env_or_settings(settings, env_name, settings_name, default):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return settings.get(settings_name, default)

c2cwsgiutils/debug.py

@@ -3,7 +3,7 @@
 import traceback
 import sys
 
-from c2cwsgiutils import _utils
+from c2cwsgiutils import _utils, _auth
 
 CONFIG_KEY = 'c2c.debug_view_secret'
 ENV_KEY = 'DEBUG_VIEW_SECRET'
@@ -12,7 +12,7 @@
 
 
 def _dump_stacks(request):
-    _utils.auth_view(request, ENV_KEY, CONFIG_KEY)
+    _auth.auth_view(request, ENV_KEY, CONFIG_KEY)
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
     code = []
     for threadId, stack in sys._current_frames().items():

c2cwsgiutils/pyramid.py

@@ -1,7 +1,7 @@
 import cornice
 import pyramid_tm
 
-from c2cwsgiutils import stats, pyramid_logging, sql_profiler, version, debug
+from c2cwsgiutils import stats_pyramid, pyramid_logging, sql_profiler, version, debug
 
 
 def includeme(config):
@@ -12,7 +12,7 @@ def includeme(config):
     config.add_settings(handle_exceptions=False)
     config.include(pyramid_tm.includeme)
     config.include(cornice.includeme)
-    stats.init(config)
+    stats_pyramid.init(config)
     pyramid_logging.install_subscriber(config)
     sql_profiler.init(config)
     version.init(config)

c2cwsgiutils/pyramid_logging.py

@@ -14,7 +14,7 @@
 import cee_syslog_handler
 from pyramid.threadlocal import get_current_request
 
-from c2cwsgiutils import _utils
+from c2cwsgiutils import _utils, _auth
 
 CONFIG_KEY = 'c2c.log_view_secret'
 ENV_KEY = 'LOG_VIEW_SECRET'
@@ -94,7 +94,7 @@ def install_subscriber(config):
 
 
 def _logging_change_level(request):
-    _utils.auth_view(request, ENV_KEY, CONFIG_KEY)
+    _auth.auth_view(request, ENV_KEY, CONFIG_KEY)
     name = request.params['name']
     level = request.params.get('level')
     logger = logging.getLogger(name)

c2cwsgiutils/sql_profiler.py

@@ -8,7 +8,7 @@
 import sqlalchemy.engine
 from threading import Lock
 
-from c2cwsgiutils import _utils
+from c2cwsgiutils import _utils, _auth
 
 ENV_KEY = 'SQL_PROFILER_SECRET'
 CONFIG_KEY = 'c2c.sql_profiler_secret'
@@ -42,7 +42,7 @@ def profile(self, conn, _cursor, statement, parameters, _context, _executemany):
 
 def _sql_profiler_view(request):
     global repository
-    _utils.auth_view(request, ENV_KEY, CONFIG_KEY)
+    _auth.auth_view(request, ENV_KEY, CONFIG_KEY)
     if 'enable' in request.params:
         if request.params['enable'] == '1':
             if repository is None:

c2cwsgiutils/stats.py

@@ -9,13 +9,9 @@
 import time
 import threading
 
-import pyramid.events
-from pyramid.httpexceptions import HTTPException
-import sqlalchemy.event
-
 from c2cwsgiutils import _utils
 
-BACKENDS = []
+BACKENDS = {}
 LOG = logging.getLogger(__name__)
 
 
@@ -47,7 +43,7 @@ def set_gauge(key, value):
     :param key: The path of the key, given as a list.
     :param value: The new value of the gauge
     """
-    for backend in BACKENDS:
+    for backend in BACKENDS.values():
         backend.gauge(key, value)
 
 
@@ -57,7 +53,7 @@ def increment_counter(key, increment=1):
     :param key: The path of the key, given as a list.
     :param increment: The increment
     """
-    for backend in BACKENDS:
+    for backend in BACKENDS.values():
         backend.counter(key, increment)
 
 
@@ -72,7 +68,7 @@ def __init__(self, key):
     def stop(self, key_final=None):
         if key_final is not None:
             self._key = key_final
-        for backend in BACKENDS:
+        for backend in BACKENDS.values():
             backend.timer(self._key, time.time() - self._start)
 
 
@@ -173,119 +169,15 @@ def counter(self, key, increment):
         self._send(message)
 
 
-def _create_finished_cb(kind, measure):  # pragma: nocover
-    def finished_cb(request):
-        if request.exception is not None:
-            if isinstance(request.exception, HTTPException):
-                status = request.exception.code
-            else:
-                status = 500
-        else:
-            status = request.response.status_code
-        if request.matched_route is None:
-            name = "_not_found"
-        else:
-            name = request.matched_route.name
-        key = [kind, request.method, name, str(status)]
-        measure.stop(key)
-    return finished_cb
-
-
-def _request_callback(event):  # pragma: nocover
-    """
-    Callback called when a new HTTP request is incoming.
-    """
-    measure = timer()
-    event.request.add_finished_callback(_create_finished_cb("route", measure))
-
-
-def _before_rendered_callback(event):  # pragma: nocover
-    """
-    Callback called when the rendering is starting.
-    """
-    request = event.get("request", None)
-    if request:
-        measure = timer()
-        request.add_finished_callback(_create_finished_cb("render", measure))
-
-
-def _simplify_sql(sql):
-    """
-    Simplify SQL statements to make them easier on the eye and shorter for the stats.
-    """
-    sql = " ".join(sql.split("\n"))
-    sql = re.sub(r"  +", " ", sql)
-    sql = re.sub(r"SELECT .*? FROM", "SELECT FROM", sql)
-    sql = re.sub(r"INSERT INTO (.*?) \(.*", r"INSERT INTO \1", sql)
-    sql = re.sub(r"SET .*? WHERE", "SET WHERE", sql)
-    sql = re.sub(r"IN \((?:%\(\w+\)\w(?:, *)?)+\)", "IN (?)", sql)
-    return re.sub(r"%\(\w+\)\w", "?", sql)
-
-
-def _before_cursor_execute(conn, _cursor, statement,
-                           _parameters, _context, _executemany):
-    measure = timer(["sql", _simplify_sql(statement)])
-
-    def after(*_args, **_kwargs):
-        measure.stop()
-
-    sqlalchemy.event.listen(conn, "after_cursor_execute", after, once=True)
-
-
-def _before_commit(session):  # pragma: nocover
-    measure = timer(["sql", "commit"])
-
-    def after(*_args, **_kwargs):
-        measure.stop()
-
-    sqlalchemy.event.listen(session, "after_commit", after, once=True)
-
-
-def init_db_spy():  # pragma: nocover
-    """
-    Subscribe to SQLAlchemy events in order to get some stats on DB interactions.
-    """
-    from sqlalchemy.engine import Engine
-    from sqlalchemy.orm import Session
-    sqlalchemy.event.listen(Engine, "before_cursor_execute", _before_cursor_execute)
-    sqlalchemy.event.listen(Session, "before_commit", _before_commit)
-
-
-def init_pyramid_spy(config):  # pragma: nocover
-    """
-    Subscribe to Pyramid events in order to get some stats on route time execution.
-    :param config: The Pyramid config
-    """
-    config.add_subscriber(_request_callback, pyramid.events.NewRequest)
-    config.add_subscriber(_before_rendered_callback, pyramid.events.BeforeRender)
-
-
-def init_backends(config):
+def init_backends(settings):
     """
     Initialize the backends according to the configuration.
     :param config: The Pyramid config
     """
-    if _utils.env_or_config(config, "STATS_VIEW", "c2c.stats_view", False):  # pragma: nocover
-        memory_backend = _MemoryBackend()
-        BACKENDS.append(memory_backend)
+    if _utils.env_or_settings(settings, "STATS_VIEW", "c2c.stats_view", False):  # pragma: nocover
+        BACKENDS['memory'] = _MemoryBackend()
 
-        config.add_route("c2c_read_stats_json", _utils.get_base_path(config) + r"/stats.json",
-                         request_method="GET")
-        config.add_view(memory_backend.get_stats, route_name="c2c_read_stats_json", renderer="json",
-                        http_cache=0)
-
-    statsd_address = _utils.env_or_config(config, "STATSD_ADDRESS", "c2c.statsd_address", None)
+    statsd_address = _utils.env_or_settings(settings, "STATSD_ADDRESS", "c2c.statsd_address", None)
     if statsd_address is not None:  # pragma: nocover
-        statsd_prefix = _utils.env_or_config(config, "STATSD_PREFIX", "c2c.statsd_prefix", "")
-        BACKENDS.append(_StatsDBackend(statsd_address, statsd_prefix))
-
-
-def init(config):
-    """
-    Initialize the whole stats module.
-    :param config: The Pyramid config
-    """
-    init_backends(config)
-    if BACKENDS:  # pragma: nocover
-        init_pyramid_spy(config)
-        init_db_spy()
+        statsd_prefix = _utils.env_or_settings(settings, "STATSD_PREFIX", "c2c.statsd_prefix", "")
+        BACKENDS['statsd'] = _StatsDBackend(statsd_address, statsd_prefix)