From 440bff65c6d7bc2df87d40b93e2b6d493a896aec Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 25 May 2023 15:31:03 +0200 Subject: [PATCH 1/2] Use separate logger for logging raw SQL queries Trying to debug log the db module *without* getting drowned in generated SQL queries is hard. Making a sub-logger at least enables us to deselect debug logging of queries wile debug logging everything else from nav.ipdevpoll.db --- python/nav/ipdevpoll/db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/nav/ipdevpoll/db.py b/python/nav/ipdevpoll/db.py index 3c511ec8ff..7c6576e9b3 100644 --- a/python/nav/ipdevpoll/db.py +++ b/python/nav/ipdevpoll/db.py @@ -29,6 +29,7 @@ from psycopg2 import InterfaceError, OperationalError _logger = logging.getLogger(__name__) +_query_logger = logging.getLogger(".".join((__name__, "query"))) class ResetDBConnectionError(Exception): @@ -51,7 +52,7 @@ def django_debug_cleanup(): if query_count: runtime = sum_django_queries_runtime() thread = threading.current_thread() - _logger.debug( + _query_logger.debug( "Thread %s/%s: Removing %d logged Django queries " "(total time %.03f):\n%s", thread.ident, @@ -117,8 +118,9 @@ def _reset(*args, **kwargs): "dropped, resetting it now - you may see further " "errors about this until the situation is fully " "resolved for all threads " - "(this thread is '%s', error was '%s')", + "(this thread is '%s', function was: %r, error was '%s')", thread.name, + func, error, ) django.db.connection.connection = None From e8416e6fde8d5b098f21d21cc97d69113c38c89b Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 1 Jun 2023 12:26:05 +0200 Subject: [PATCH 2/2] Observe Twisted logs as well The Twisted library doesn't use the regular Python logging framework. However, a twisted log observer can be added to propagate twisted log messages to the regular python logging framework, which is what the rest of ipdevpoll process uses. --- python/nav/ipdevpoll/daemon.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/nav/ipdevpoll/daemon.py b/python/nav/ipdevpoll/daemon.py index 96cda4f94f..dde08e32d6 100644 --- a/python/nav/ipdevpoll/daemon.py +++ b/python/nav/ipdevpoll/daemon.py @@ -30,6 +30,7 @@ import time import argparse +import twisted from twisted.internet import reactor from twisted.internet.defer import maybeDeferred, setDebugging from twisted.python.failure import Failure @@ -412,6 +413,9 @@ def run(self): def init_logging(self, stderr_only=False): """Initializes ipdevpoll logging for the current process.""" + observer = twisted.python.log.PythonLoggingObserver() + observer.start() + formatter = ContextFormatter(self.options.pidlog) logfile_name = None