Improvements to the way we are setting log level for httpx and httpco…

…re libraries instead of dictConfig (#397)

Co-authored-by: Nithin Bodanapu <[email protected]>

cognite/extractorutils/_inner_util.py

@@ -25,6 +25,17 @@ def _resolve_log_level(level: str) -> int:
     return {"NOTSET": 0, "DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}[level.upper()]
 
 
+def resolve_log_level_for_httpx(level: str) -> str:
+    return {
+        None: "WARNING",
+        "INFO": "WARNING",
+        "WARNING": "WARNING",
+        "ERROR": "ERROR",
+        "CRITICAL": "CRITICAL",
+        "DEBUG": "DEBUG",
+    }.get(level, "WARNING")
+
+
 class _DecimalEncoder(json.JSONEncoder):
     def default(self, obj: Any) -> Dict[str, str]:
         if isinstance(obj, Decimal):

cognite/extractorutils/configtools/elements.py

@@ -32,6 +32,7 @@
     OAuthClientCredentials,
 )
 from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline
+from cognite.extractorutils._inner_util import resolve_log_level_for_httpx
 from cognite.extractorutils.configtools._util import _load_certificate_data
 from cognite.extractorutils.exceptions import InvalidConfigError
 from cognite.extractorutils.metrics import (
@@ -491,6 +492,14 @@ def setup_logging(self, suppress_console: bool = False) -> None:
             if root.getEffectiveLevel() > file_handler.level:
                 root.setLevel(file_handler.level)
 
+            log_level = logging.getLevelName(root.getEffectiveLevel())
+            httpx_log_level = resolve_log_level_for_httpx(log_level)
+            httpx_logger = logging.getLogger("httpx")
+            httpx_logger.setLevel(httpx_log_level)
+
+            http_core_logger = logging.getLogger("httpcore")
+            http_core_logger.setLevel(httpx_log_level)
+
 
 @dataclass
 class _PushGatewayConfig: