From c50cc032316e95356db4d129db85702f87a9c65b Mon Sep 17 00:00:00 2001
From: extreme4all <40169115+extreme4all@users.noreply.github.com>
Date: Thu, 21 Nov 2024 23:27:28 +0100
Subject: [PATCH 1/2] improve logs

---
 mysql/docker-entrypoint-initdb.d/02_data.sql | 17 +++++-
 src/core/database/database.py                |  2 +-
 src/core/fastapi/middleware/logging.py       |  2 +-
 src/core/logging_config.py                   | 64 ++++++++++----------
 4 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/mysql/docker-entrypoint-initdb.d/02_data.sql b/mysql/docker-entrypoint-initdb.d/02_data.sql
index 0b4e5931..8cadf5cf 100644
--- a/mysql/docker-entrypoint-initdb.d/02_data.sql
+++ b/mysql/docker-entrypoint-initdb.d/02_data.sql
@@ -272,4 +272,19 @@ SET
     name=replace(name,'-',' '),
     normalized_name=replace(name,'-',' ')
 WHERE name LIKE 'anonymoususer%'
-;
\ No newline at end of file
+;
+INSERT INTO `report_sighting` (`reporting_id`, `reported_id`, `manual_detect`)
+VALUES
+    (1, 2, 0),
+    (1, 3, 0),
+    (1, 5, 0),
+    (2, 3, 0),
+    (3, 4, 0),
+    (4, 5, 1),
+    (5, 6, 0),
+    (6, 7, 0),
+    (7, 8, 1),
+    (8, 9, 0),
+    (9, 10, 1),
+    (10, 11, 0)
+;
diff --git a/src/core/database/database.py b/src/core/database/database.py
index b02fb3f8..4c55a1fb 100644
--- a/src/core/database/database.py
+++ b/src/core/database/database.py
@@ -8,7 +8,7 @@
     settings.DATABASE_URL,
     pool_timeout=settings.POOL_TIMEOUT,
     pool_recycle=settings.POOL_RECYCLE,
-    echo=(settings.ENV != "PRD"),
+    # echo=(settings.ENV != "PRD"),
     pool_pre_ping=True,
 )
 
diff --git a/src/core/fastapi/middleware/logging.py b/src/core/fastapi/middleware/logging.py
index 34113fe6..fe1fed6f 100644
--- a/src/core/fastapi/middleware/logging.py
+++ b/src/core/fastapi/middleware/logging.py
@@ -18,7 +18,7 @@ async def dispatch(self, request: Request, call_next):
             for key, value in request.query_params.items()
         ]
 
-        logger.debug(
+        logger.info(
             {
                 "url": request.url.path,
                 "params": query_params_list,
diff --git a/src/core/logging_config.py b/src/core/logging_config.py
index 0bb99940..d66a2a1a 100644
--- a/src/core/logging_config.py
+++ b/src/core/logging_config.py
@@ -1,40 +1,42 @@
 import json
 import logging
-import sys
-
-# # log formatting
-formatter = logging.Formatter(
-    json.dumps(
-        {
-            "ts": "%(asctime)s",
-            "name": "%(name)s",
-            "function": "%(funcName)s",
-            "level": "%(levelname)s",
-            "msg": json.dumps("%(message)s"),
+
+
+# Configure JSON logging
+class JsonFormatter(logging.Formatter):
+    def format(self, record):
+        log_record = {
+            "ts": self.formatTime(record, self.datefmt),
+            "lvl": record.levelname,
+            "module": record.module,
+            "func": record.funcName,
+            "line": record.lineno,
+            "msg": record.getMessage(),
         }
-    )
-)
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_record)
+
+
+class IgnoreSQLWarnings(logging.Filter):
+    def filter(self, record):
+        ignore_messages = ["Unknown table", "Duplicate entry"]
+        # Check if any of the ignore messages are in the log record message
+        if any(msg in record.getMessage() for msg in ignore_messages):
+            return False  # Don't log
+        return True  # Log
 
-stream_handler = logging.StreamHandler(sys.stdout)
 
-stream_handler.setFormatter(formatter)
+# Set up the logger
+handler = logging.StreamHandler()
+handler.setFormatter(JsonFormatter())
 
-handlers = [stream_handler]
+logging.basicConfig(level=logging.INFO, handlers=[handler])
 
-logging.basicConfig(level=logging.DEBUG, handlers=handlers)
 
 # set imported loggers to warning
-logging.getLogger("urllib3").setLevel(logging.DEBUG)
-logging.getLogger("uvicorn").setLevel(logging.DEBUG)
-logging.getLogger("aiomysql").setLevel(logging.ERROR)
-logging.getLogger("aiokafka").setLevel(logging.WARNING)
-
-# if settings.ENV == "PRD":
-#     uvicorn_error = logging.getLogger("uvicorn.error")
-#     uvicorn_error.disabled = True
-#     uvicorn_access = logging.getLogger("uvicorn.access")
-#     uvicorn_access.disabled = True
-
-# # https://github.com/aio-libs/aiomysql/issues/103
-# # https://github.com/coleifer/peewee/issues/2229
-# warnings.filterwarnings("ignore", ".*Duplicate entry.*")
+logging.getLogger("asyncmy").addFilter(IgnoreSQLWarnings())
+# logging.getLogger("urllib3").setLevel(logging.DEBUG)
+# logging.getLogger("uvicorn").setLevel(logging.DEBUG)
+# logging.getLogger("aiomysql").setLevel(logging.ERROR)
+# logging.getLogger("aiokafka").setLevel(logging.WARNING)

From caa35cb0930009362fb78462a4937385717cc431 Mon Sep 17 00:00:00 2001
From: extreme4all <40169115+extreme4all@users.noreply.github.com>
Date: Thu, 21 Nov 2024 23:41:23 +0100
Subject: [PATCH 2/2] json logs

---
 src/core/__init__.py                       | 4 ++--
 src/core/{logging_config.py => logging.py} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename src/core/{logging_config.py => logging.py} (100%)

diff --git a/src/core/__init__.py b/src/core/__init__.py
index 170e7bd3..18d7a41f 100644
--- a/src/core/__init__.py
+++ b/src/core/__init__.py
@@ -1,3 +1,3 @@
-from . import config, logging_config
+from . import config, logging
 
-__all__ = ["logging_config", "config"]
+__all__ = ["logging", "config"]
diff --git a/src/core/logging_config.py b/src/core/logging.py
similarity index 100%
rename from src/core/logging_config.py
rename to src/core/logging.py