From 2bf7b65df57e80a34f2df047e9adda625137624a Mon Sep 17 00:00:00 2001 From: yinjiaqi Date: Wed, 1 Jan 2025 20:38:57 +0800 Subject: [PATCH 1/6] =?UTF-8?q?SDK=E6=96=B0=E5=A2=9E=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/BasisModule/Trace/Debug.md | 32 +++++ python/__init__.py | 3 + python/tests/test_log_set_log_config.py | 109 +++++++++++++++ python/utils/logger_file_headler.py | 91 +++++++++++++ python/utils/logger_util.py | 170 ++++++++++++++++++++---- 5 files changed, 380 insertions(+), 25 deletions(-) create mode 100644 python/tests/test_log_set_log_config.py create mode 100644 python/utils/logger_file_headler.py diff --git a/docs/BasisModule/Trace/Debug.md b/docs/BasisModule/Trace/Debug.md index 11d4239e3..3376bfb01 100644 --- a/docs/BasisModule/Trace/Debug.md +++ b/docs/BasisModule/Trace/Debug.md @@ -51,4 +51,36 @@ System.setProperty("APPBUILDER_LOGLFILE", "/tmp/appbuilder.log"); ```golang // golang os.Setenv("APPBUILDER_LOGLEVEL", "/tmp/appbuilder.log") +``` + +## `setLogConfig`功能 + +Appbuilder-SDK新增滚动日志功能 + +主要参数: +- console_show: 数据类型bool，默认值True，LOG日志是否在控制台输出 +- loglevel: 数据类型str，默认值"DEBUG"，LOG日志级别 +- file_name: 数据类型str，默认值"tmp.log"，LOG日志名称 +- when: 数据类型str，默认值"MIDNIGHT"，LOG日志滚动更新时间单位 + - "S": 以秒为单位 + - "M": 以分钟为单位 + - "H": 以小时为单位 + - "D": 以天为时间单位 + - "MIDNIGHT": 每日凌晨更新 +- interval: 数据类型int，默认值1，LOG日志按时间滚动的参数，默认值为1，与when参数联合使用 +- max_bytes: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，单个滚动的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 即需要传入 # 以B为单位 +- total_size_limit: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 # 以B为单位 +- backup_count: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大数量 + +```python +# python +appbuilder.logger.setLogConfig( + console_show = False, + file_name="appbuilder.log", + when="MIDNIGHT", # 每日凌晨更新 + interval=1, + max_bytes=100 * 1024 *1024, # 最大日志大小为100MB + total_size_limit=1024 * 1024 *1024, # 最大储存1GB的日志 + backup_count=10, # 当前目录储存的最大LOG日志数 + ) ``` \ No newline at end of file diff --git a/python/__init__.py b/python/__init__.py index 09fb670ae..8f0ca46c3 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -203,8 +203,11 @@ def get_default_header(): from appbuilder.utils.trace.tracer import AppBuilderTracer, AppbuilderInstrumentor +from .utils.logger_file_headler import SizeAndTimeRotatingFileHandler + __all__ = [ "logger", + "SizeAndTimeRotatingFileHandler", "BadRequestException", "ForbiddenException", "NotFoundException", diff --git a/python/tests/test_log_set_log_config.py b/python/tests/test_log_set_log_config.py new file mode 100644 index 000000000..ca63cd873 --- /dev/null +++ b/python/tests/test_log_set_log_config.py @@ -0,0 +1,109 @@ +# Copyright (c) 2024 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import time +import logging +import unittest + + +from appbuilder import SizeAndTimeRotatingFileHandler +from appbuilder.utils.logger_util import LoggerWithLoggerId + +class TestLogSetLogConfig(unittest.TestCase): + def test_set_log_config(self): + lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') + lwl.setLogConfig( + console_show = True, + loglevel='DEBUG', + file_name='test.log', + when='D', + interval=0, # 测试interval<1时，自动更新为1 + max_bytes=None, # 测试not max_bytes or max_bytes <= 0时，自动更新为sys.maxsize + total_size_limit=None, # 测试not total_size_limit or total_size_limit <= 0时，自动更新为sys.maxsize + backup_count=None, # 测试not backup_count or backup_count <= 0时，自动更新为sys.maxsize + ) + + def test_set_log_config_raise_error(self): + lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') + with self.assertRaises(ValueError): + lwl.setLogConfig( + console_show = True, + loglevel='DEBUG', + file_name='test.log', + when='ERROR-WHEN', + interval=0, # 测试interval<1时，自动更新为1 + max_bytes=None, # 测试not max_bytes or max_bytes <= 0时，自动更新为sys.maxsize + total_size_limit=None, # 测试not total_size_limit or total_size_limit <= 0时，自动更新为sys.maxsize + backup_count=None, # 测试not backup_count or backup_count <= 0时，自动更新为sys.maxsize + ) + + def test_rolling_with_time(self): + logger = logging.getLogger('CustomLogger') + logger.setLevel(logging.DEBUG) + handler = SizeAndTimeRotatingFileHandler( + filename ='test.log', + when='S', + interval=1, + max_bytes=1024*100*1024, + backup_count=10, + total_size_limit=1024*300*1024 + ) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + for i in range(3): + logger.info("This is a test log message.") + time.sleep(1) + + def test_rolling_with_size(self): + logger = logging.getLogger('CustomLogger') + logger.setLevel(logging.DEBUG) + handler = SizeAndTimeRotatingFileHandler( + filename ='test.log', + when='S', + interval=10, + max_bytes=1*1024, + backup_count=2, + total_size_limit=1024*300*1024 + ) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + for i in range(100): + logger.info("This is a test log message."*100) + time.sleep(0.001) + + def test_rolling_to_total_max_size(self): + logger = logging.getLogger('CustomLogger') + logger.setLevel(logging.DEBUG) + handler = SizeAndTimeRotatingFileHandler( + filename ='test.log', + when='S', + interval=100, + max_bytes=10*1024, + backup_count=10000, + total_size_limit=20*1024 + ) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + for i in range(100): + logger.info("This is a test log message."*100) + time.sleep(0.001) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/python/utils/logger_file_headler.py b/python/utils/logger_file_headler.py new file mode 100644 index 000000000..dc310ce75 --- /dev/null +++ b/python/utils/logger_file_headler.py @@ -0,0 +1,91 @@ +# Copyright (c) 2024 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import time +import glob +import logging +from datetime import datetime, timedelta + +class SizeAndTimeRotatingFileHandler(logging.Handler): + def __init__(self, filename, when='S', interval=1, max_bytes=0, backup_count=0, total_size_limit=0): + super().__init__() + self.base_filename = filename + self.when = when.upper() + self.interval = interval + self.max_bytes = max_bytes + self.backup_count = backup_count + self.total_size_limit = total_size_limit + self.current_time = datetime.now() + self.current_file = self.base_filename + self.stream = open(self.current_file, 'a') + self.last_rollover = time.time() + + def _get_new_filename(self): + suffix = self.current_time.strftime("%Y-%m-%d_%H-%M-%S") + return f"{self.base_filename}.{suffix}" + + def emit(self, record): + if self.shouldRollover(record): + self.doRollover() + self.stream.write(self.format(record) + '\n') + self.stream.flush() + + def shouldRollover(self, record): + current_time = time.time() + current_size = os.path.getsize(self.current_file) + + time_rollover = False + if self.when == 'S': + time_rollover = current_time >= self.last_rollover + self.interval + elif self.when == 'M': + time_rollover = current_time >= self.last_rollover + self.interval * 60 + elif self.when == 'H': + time_rollover = current_time >= self.last_rollover + self.interval * 3600 + elif self.when == 'D': + time_rollover = current_time >= self.last_rollover + self.interval * 86400 + elif self.when == 'MIDNIGHT': + time_rollover = datetime.fromtimestamp(current_time).date() != datetime.fromtimestamp(self.last_rollover).date() + + size_rollover = current_size >= self.max_bytes if self.max_bytes > 0 else False + + return time_rollover or size_rollover + + def doRollover(self): + self.stream.close() + self.current_time = datetime.now() + new_filename = self._get_new_filename() + os.rename(self.current_file, new_filename) # Rename current file to new name + self.current_file = self.base_filename + self.stream = open(self.current_file, 'a') + self.last_rollover = time.time() + self.manage_log_files() + + def manage_log_files(self): + log_files = sorted(glob.glob(f"{self.base_filename}.*"), key=os.path.getmtime) + + while len(log_files) > self.backup_count: + oldest_log = log_files.pop(0) + os.remove(oldest_log) + + while self._total_size(log_files) > self.total_size_limit: + if log_files: + oldest_log = log_files.pop(0) + os.remove(oldest_log) + + def _total_size(self, files): + return sum(os.path.getsize(f) for f in files if os.path.exists(f)) + + def close(self): + self.stream.close() + super().close() \ No newline at end of file diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index 3396567d4..e3766c22f 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -16,13 +16,15 @@ """ 日志 """ -import uuid import os import sys +import uuid +import logging +import logging.handlers import logging.config from threading import current_thread - - +from typing import Optional + LOGGING_CONFIG = { "version": 1, "disable_existing_loggers": False, @@ -31,29 +33,60 @@ "format": "[%(asctime)s.%(msecs)03d] %(filename)s [line:%(lineno)d] %(levelname)s [%(logid)s] %(message)s", }, }, - "handlers": { - "console": { - "level": "INFO", - "class": "logging.StreamHandler", - "formatter": "standard", - "stream": "ext://sys.stdout", # Use standard output - }, - "file": { - "level": "INFO", - "class": "logging.FileHandler", - "filename": "tmp.log", - "formatter": "standard", - }, - }, + "handlers": {}, "loggers": { "appbuilder": { - "handlers": ["console"], + "handlers": [], "level": "INFO", "propagate": True, }, }, } +CONSOLE_HEADER = { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "standard", + "stream": "ext://sys.stdout", # Use standard output +} + +ERROR_FILE_HEADER = { + "level": "ERROR", + "class": "logging.FileHandler", + "filename": "error.tmp.log", + "formatter": "standard", +} + +FILE_HEADER = { + "level": "DEBUG", + "class": "logging.FileHandler", + "filename": "tmp.log", + "formatter": "standard", +} + +ERROR_SET_CONFIG_HEADER = { + 'level': 'ERROR', + 'formatter': 'standard', + 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', + 'filename': 'error.tmp.log', + 'when': 'MIDDNIGHT', + 'interval': 1, + 'max_bytes': 5*1024*1024, + 'backup_count': 20, + 'total_size_limit': 100*1024*1024 +} + +SET_CONFIG_HEADER = { + 'level': 'DEBUG', + 'formatter': 'standard', + 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', + 'filename': 'tmp.log', + 'when': 'MIDNIGHT', + 'interval': 1, + 'max_bytes': 5*1024*1024, + 'backup_count': 20, + 'total_size_limit': 100*1024*1024 +} class LoggerWithLoggerId(logging.LoggerAdapter): """ @@ -63,12 +96,21 @@ def __init__(self, logger, extra, loglevel): """ init """ + LOGGING_CONFIG["handlers"] = {} + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] log_file = os.environ.get("APPBUILDER_LOGFILE", "") if log_file: - LOGGING_CONFIG["handlers"]["file"]["filename"] = log_file + ERROR_FILE_HEADER["filename"] = f"error.{log_file}" + FILE_HEADER["filename"] = log_file + FILE_HEADER["level"] = loglevel + LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") - LOGGING_CONFIG["handlers"]["file"]["level"] = loglevel - LOGGING_CONFIG['handlers']['console']['level'] = loglevel + if loglevel in ("DEBUG", "INFO", "WARNING"): + LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + CONSOLE_HEADER["level"] = loglevel + LOGGING_CONFIG["handlers"]["console"] = CONSOLE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("console") LOGGING_CONFIG['loggers']['appbuilder']['level'] = loglevel logging.config.dictConfig(LOGGING_CONFIG) logging.LoggerAdapter.__init__(self, logger, extra) @@ -107,8 +149,17 @@ def setFilename(self, filename): set filename """ if "file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + FILE_HEADER["filename"] = filename + LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") - LOGGING_CONFIG["handlers"]["file"]["filename"] = filename + if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + ERROR_FILE_HEADER["filename"] = f"error.{filename}" + LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + FILE_HEADER["filename"] = filename + ERROR_FILE_HEADER["filename"] = f"error.{filename}" + LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER + LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER logging.config.dictConfig(LOGGING_CONFIG) def setLoglevel(self, level): @@ -116,13 +167,82 @@ def setLoglevel(self, level): set log level """ log_level = level.strip().lower() - if log_level not in ["debug", "info", "warning", "error"]: raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % log_level) log_level = log_level.upper() - LOGGING_CONFIG['handlers']['console']['level'] = log_level + if "file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + FILE_HEADER["level"] = log_level + LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER + if "console" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] or not LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + CONSOLE_HEADER["level"] = log_level + LOGGING_CONFIG["handlers"]["console"] = CONSOLE_HEADER LOGGING_CONFIG['loggers']['appbuilder']['level'] = log_level - LOGGING_CONFIG["handlers"]["file"]["level"] = log_level + logging.config.dictConfig(LOGGING_CONFIG) + + def setLogConfig(self, + console_show: bool = True, + loglevel: str = "DEBUG", + file_name: str = "tmp.log", + when: str = "MIDNIGHT", + interval: int = 1, + max_bytes: Optional[int] = None, # 以B为单位 + total_size_limit: Optional[int] = None, # 以B为单位 + backup_count: Optional[int] = None + ): + LOGGING_CONFIG["handlers"] = {} + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] + + + # 设置console输出日志 + if console_show: + CONSOLE_HEADER['level'] = loglevel + LOGGING_CONFIG["handlers"]["console"] = CONSOLE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("console") + + # 参数验证 + if not max_bytes or max_bytes <= 0: + max_bytes = sys.maxsize + if not total_size_limit or total_size_limit <= 0: + total_size_limit = sys.maxsize + if not backup_count or backup_count <= 0: + backup_count = sys.maxsize + if interval < 1: + interval = 1 + when = when.strip().lower() + if when not in ["s", "m", "h", "d", "midnight"]: + raise ValueError("expected when in [S, M, H, D, MIDNIGHT], but got %s" % when) + + # 设置文件输出日志 + # 设置日志级别 + SET_CONFIG_HEADER['level'] = loglevel + + # 设置文件名称 + SET_CONFIG_HEADER['filename'] = file_name + ERROR_SET_CONFIG_HEADER['filename'] = f"error.{file_name}" + + # 设置滚动时间 + SET_CONFIG_HEADER['when'] = when + ERROR_SET_CONFIG_HEADER['when'] = when + SET_CONFIG_HEADER['interval'] = interval + ERROR_SET_CONFIG_HEADER['interval'] = interval + + # 设置最大文件大小 + + SET_CONFIG_HEADER['max_bytes'] = max_bytes + ERROR_SET_CONFIG_HEADER['max_bytes'] = max_bytes + + # 设置总大小限制 + SET_CONFIG_HEADER['total_size_limit'] = total_size_limit + ERROR_SET_CONFIG_HEADER['total_size_limit'] = total_size_limit + + # 设置备份数量 + SET_CONFIG_HEADER['backup_count'] = backup_count + ERROR_SET_CONFIG_HEADER['backup_count'] = backup_count + + LOGGING_CONFIG["handlers"]["file"] = SET_CONFIG_HEADER + LOGGING_CONFIG["handlers"]["error_file"] = ERROR_SET_CONFIG_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].extend(["file", "error_file"]) + LOGGING_CONFIG['loggers']['appbuilder']['level'] = loglevel logging.config.dictConfig(LOGGING_CONFIG) def process(self, msg, kwargs): From c21acbd71a8ac36222d136c56f8174621346eff5 Mon Sep 17 00:00:00 2001 From: yinjiaqi Date: Wed, 1 Jan 2025 20:51:10 +0800 Subject: [PATCH 2/6] update-error-filename --- python/utils/logger_util.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index e3766c22f..29a723ff5 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -100,7 +100,7 @@ def __init__(self, logger, extra, loglevel): LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] log_file = os.environ.get("APPBUILDER_LOGFILE", "") if log_file: - ERROR_FILE_HEADER["filename"] = f"error.{log_file}" + ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(log_file) FILE_HEADER["filename"] = log_file FILE_HEADER["level"] = loglevel LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER @@ -153,11 +153,11 @@ def setFilename(self, filename): LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: - ERROR_FILE_HEADER["filename"] = f"error.{filename}" + ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(filename) LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") FILE_HEADER["filename"] = filename - ERROR_FILE_HEADER["filename"] = f"error.{filename}" + ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(filename) LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER logging.config.dictConfig(LOGGING_CONFIG) @@ -218,7 +218,7 @@ def setLogConfig(self, # 设置文件名称 SET_CONFIG_HEADER['filename'] = file_name - ERROR_SET_CONFIG_HEADER['filename'] = f"error.{file_name}" + ERROR_SET_CONFIG_HEADER['filename'] = _add_error_to_file_name(file_name) # 设置滚动时间 SET_CONFIG_HEADER['when'] = when @@ -309,5 +309,11 @@ def get_logger(name, level=logging.INFO): logger.propagate = False return logger +def _add_error_to_file_name(filename): + prefix = "error." + dir_name, base_name = os.path.split(filename) + new_base_name = f"{prefix}{base_name}" + return os.path.join(dir_name, new_base_name) + logger = _setup_logging() From df45a5c48f430e2b5990cfeb9bf5fb9380f7a0bf Mon Sep 17 00:00:00 2001 From: yinjiaqi Date: Thu, 2 Jan 2025 09:58:29 +0800 Subject: [PATCH 3/6] update --- python/tests/test_log_set_log_config.py | 34 +++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/python/tests/test_log_set_log_config.py b/python/tests/test_log_set_log_config.py index ca63cd873..e5279f105 100644 --- a/python/tests/test_log_set_log_config.py +++ b/python/tests/test_log_set_log_config.py @@ -49,23 +49,25 @@ def test_set_log_config_raise_error(self): ) def test_rolling_with_time(self): - logger = logging.getLogger('CustomLogger') - logger.setLevel(logging.DEBUG) - handler = SizeAndTimeRotatingFileHandler( - filename ='test.log', - when='S', - interval=1, - max_bytes=1024*100*1024, - backup_count=10, - total_size_limit=1024*300*1024 - ) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) + time_msgs = ['S', 'M', 'H', 'D', 'MIDNIGHT'] + for time_msg in time_msgs: + logger = logging.getLogger('CustomLogger') + logger.setLevel(logging.DEBUG) + handler = SizeAndTimeRotatingFileHandler( + filename ='test.log', + when=time_msg, + interval=1, + max_bytes=1024*100*1024, + backup_count=10, + total_size_limit=1024*300*1024 + ) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) - for i in range(3): - logger.info("This is a test log message.") - time.sleep(1) + for _ in range(2): + logger.info("This is a test log message.") + time.sleep(0.1) def test_rolling_with_size(self): logger = logging.getLogger('CustomLogger') From b2d2df157243efd23ee6f5064248ed943d4f56b8 Mon Sep 17 00:00:00 2001 From: yinjiaqi Date: Thu, 2 Jan 2025 17:28:59 +0800 Subject: [PATCH 4/6] update --- docs/BasisModule/Trace/Debug.md | 26 ++++++----- python/tests/test_log_set_log_config.py | 37 ++++++++++----- python/utils/logger_util.py | 62 ++++++++++++++----------- 3 files changed, 73 insertions(+), 52 deletions(-) diff --git a/docs/BasisModule/Trace/Debug.md b/docs/BasisModule/Trace/Debug.md index 3376bfb01..a09fd0e91 100644 --- a/docs/BasisModule/Trace/Debug.md +++ b/docs/BasisModule/Trace/Debug.md @@ -58,29 +58,31 @@ os.Setenv("APPBUILDER_LOGLEVEL", "/tmp/appbuilder.log") Appbuilder-SDK新增滚动日志功能 主要参数: -- console_show: 数据类型bool，默认值True，LOG日志是否在控制台输出 +- console_output: 数据类型bool，默认值True，LOG日志是否在控制台输出 - loglevel: 数据类型str，默认值"DEBUG"，LOG日志级别 - file_name: 数据类型str，默认值"tmp.log"，LOG日志名称 -- when: 数据类型str，默认值"MIDNIGHT"，LOG日志滚动更新时间单位 +- rotate_frequency: 数据类型str，默认值"MIDNIGHT"，LOG日志滚动更新时间单位 - "S": 以秒为单位 - "M": 以分钟为单位 - "H": 以小时为单位 - "D": 以天为时间单位 - "MIDNIGHT": 每日凌晨更新 -- interval: 数据类型int，默认值1，LOG日志按时间滚动的参数，默认值为1，与when参数联合使用 -- max_bytes: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，单个滚动的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 即需要传入 # 以B为单位 -- total_size_limit: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 # 以B为单位 -- backup_count: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大数量 +- rotate_interval: 数据类型int，默认值1，LOG日志按时间滚动的参数，默认值为1，与when参数联合使用 +- max_file_size: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，单个滚动的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 即需要传入 # 以B为单位 +- total_log_size: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大大小，例：10M即为10\*1024\*1024 # 以B为单位 +- max_log_files: 数据类型Optional[int]，默认值None，传入`None`或负数会自动更新为系统最大整数`sys.maxsize`，当前目录下可储存的LOG日志文件的最大数量 +**注意:`setLogConfig`会自动生成error.file_name日志与file_name日志文件分别储存`error`级别日志和`loglevel`级别的日志,且两种日志文件的滚动逻辑是独立的，不相互影响。** ```python # python appbuilder.logger.setLogConfig( - console_show = False, + console_output = False, + loglevel="DEBUG" file_name="appbuilder.log", - when="MIDNIGHT", # 每日凌晨更新 - interval=1, - max_bytes=100 * 1024 *1024, # 最大日志大小为100MB - total_size_limit=1024 * 1024 *1024, # 最大储存1GB的日志 - backup_count=10, # 当前目录储存的最大LOG日志数 + rotate_frequency="MIDNIGHT", # 每日凌晨更新 + rotate_interval=1, + max_file_size=100 * 1024 *1024, # 最大日志大小为100MB + total_log_size=1024 * 1024 *1024, # 最大储存1GB的日志 + max_log_files=10, # 当前目录储存的最大LOG日志数 ) ``` \ No newline at end of file diff --git a/python/tests/test_log_set_log_config.py b/python/tests/test_log_set_log_config.py index e5279f105..155ded91b 100644 --- a/python/tests/test_log_set_log_config.py +++ b/python/tests/test_log_set_log_config.py @@ -24,29 +24,42 @@ class TestLogSetLogConfig(unittest.TestCase): def test_set_log_config(self): lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') lwl.setLogConfig( - console_show = True, + console_output = True, loglevel='DEBUG', file_name='test.log', - when='D', - interval=0, # 测试interval<1时，自动更新为1 - max_bytes=None, # 测试not max_bytes or max_bytes <= 0时，自动更新为sys.maxsize - total_size_limit=None, # 测试not total_size_limit or total_size_limit <= 0时，自动更新为sys.maxsize - backup_count=None, # 测试not backup_count or backup_count <= 0时，自动更新为sys.maxsize + rotate_frequency='D', + rotate_interval=0, # 测试rotate_interval<1时，自动更新为1 + max_file_size=None, # 测试not max_file_size or max_file_size <= 0时，自动更新为sys.maxsize + total_log_size=None, # 测试not total_log_size or total_log_size <= 0时，自动更新为sys.maxsize + max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize ) def test_set_log_config_raise_error(self): lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') with self.assertRaises(ValueError): lwl.setLogConfig( - console_show = True, + console_output = True, loglevel='DEBUG', file_name='test.log', - when='ERROR-WHEN', - interval=0, # 测试interval<1时，自动更新为1 - max_bytes=None, # 测试not max_bytes or max_bytes <= 0时，自动更新为sys.maxsize - total_size_limit=None, # 测试not total_size_limit or total_size_limit <= 0时，自动更新为sys.maxsize - backup_count=None, # 测试not backup_count or backup_count <= 0时，自动更新为sys.maxsize + rotate_frequency='ERROR-FREQUENCY', + rotate_interval=0, # 测试rotate_interval<1时，自动更新为1 + max_file_size=None, # 测试not max_file_size or max_file_size <= 0时，自动更新为sys.maxsize + total_log_size=None, # 测试not total_log_size or total_log_size <= 0时，自动更新为sys.maxsize + max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize ) + + with self.assertRaises(ValueError): + lwl.setLogConfig( + console_output = True, + loglevel='ERROR-LEVEL', + file_name='test.log', + rotate_frequency='D', + rotate_interval=0, # 测试rotate_interval<1时，自动更新为1 + max_file_size=0, # 测试not max_file_size or max_file_size <= 0时，自动更新为sys.maxsize + total_log_size=None, # 测试not total_log_size or total_log_size <= 0时，自动更新为sys.maxsize + max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize + ) + def test_rolling_with_time(self): time_msgs = ['S', 'M', 'H', 'D', 'MIDNIGHT'] diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index 29a723ff5..bef94fe71 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -180,37 +180,43 @@ def setLoglevel(self, level): logging.config.dictConfig(LOGGING_CONFIG) def setLogConfig(self, - console_show: bool = True, + console_output: bool = True, loglevel: str = "DEBUG", file_name: str = "tmp.log", - when: str = "MIDNIGHT", - interval: int = 1, - max_bytes: Optional[int] = None, # 以B为单位 - total_size_limit: Optional[int] = None, # 以B为单位 - backup_count: Optional[int] = None + rotate_frequency: str = "MIDNIGHT", + rotate_interval: int = 1, + max_file_size: Optional[int] = None, # 以B为单位 + total_log_size: Optional[int] = None, # 以B为单位 + max_log_files: Optional[int] = None ): LOGGING_CONFIG["handlers"] = {} LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] + # log_level 数据校验 + log_level = loglevel.strip().lower() + if log_level not in ["debug", "info", "warning", "error"]: + raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % log_level) # 设置console输出日志 - if console_show: + if console_output: CONSOLE_HEADER['level'] = loglevel LOGGING_CONFIG["handlers"]["console"] = CONSOLE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("console") + else: + LOGGING_CONFIG["loggers"]["appbuilder"]["propagate"] = False # 参数验证 - if not max_bytes or max_bytes <= 0: - max_bytes = sys.maxsize - if not total_size_limit or total_size_limit <= 0: - total_size_limit = sys.maxsize - if not backup_count or backup_count <= 0: - backup_count = sys.maxsize - if interval < 1: - interval = 1 - when = when.strip().lower() - if when not in ["s", "m", "h", "d", "midnight"]: - raise ValueError("expected when in [S, M, H, D, MIDNIGHT], but got %s" % when) + if not max_file_size or max_file_size <= 0: + max_file_size = sys.maxsize + if not total_log_size or total_log_size <= 0: + total_log_size = sys.maxsize + if not max_log_files or max_log_files <= 0: + max_log_files = sys.maxsize + if rotate_interval < 1: + rotate_interval = 1 + rotate_frequency = rotate_frequency.strip().lower() + if rotate_frequency not in ["s", "m", "h", "d", "midnight"]: + raise ValueError("expected rotate_frequency in [S, M, H, D, MIDNIGHT], but got %s" % rotate_frequency) # 设置文件输出日志 # 设置日志级别 @@ -221,23 +227,23 @@ def setLogConfig(self, ERROR_SET_CONFIG_HEADER['filename'] = _add_error_to_file_name(file_name) # 设置滚动时间 - SET_CONFIG_HEADER['when'] = when - ERROR_SET_CONFIG_HEADER['when'] = when - SET_CONFIG_HEADER['interval'] = interval - ERROR_SET_CONFIG_HEADER['interval'] = interval + SET_CONFIG_HEADER['when'] = rotate_frequency + ERROR_SET_CONFIG_HEADER['when'] = rotate_frequency + SET_CONFIG_HEADER['interval'] = rotate_interval + ERROR_SET_CONFIG_HEADER['interval'] = rotate_interval # 设置最大文件大小 - SET_CONFIG_HEADER['max_bytes'] = max_bytes - ERROR_SET_CONFIG_HEADER['max_bytes'] = max_bytes + SET_CONFIG_HEADER['max_bytes'] = max_file_size + ERROR_SET_CONFIG_HEADER['max_bytes'] = max_file_size # 设置总大小限制 - SET_CONFIG_HEADER['total_size_limit'] = total_size_limit - ERROR_SET_CONFIG_HEADER['total_size_limit'] = total_size_limit + SET_CONFIG_HEADER['total_size_limit'] = total_log_size + ERROR_SET_CONFIG_HEADER['total_size_limit'] = total_log_size # 设置备份数量 - SET_CONFIG_HEADER['backup_count'] = backup_count - ERROR_SET_CONFIG_HEADER['backup_count'] = backup_count + SET_CONFIG_HEADER['backup_count'] = max_log_files + ERROR_SET_CONFIG_HEADER['backup_count'] = max_log_files LOGGING_CONFIG["handlers"]["file"] = SET_CONFIG_HEADER LOGGING_CONFIG["handlers"]["error_file"] = ERROR_SET_CONFIG_HEADER From 6fb10ec4a7b392345db842e59eed1402d3f2ab24 Mon Sep 17 00:00:00 2001 From: yinjiaqi Date: Thu, 2 Jan 2025 19:24:55 +0800 Subject: [PATCH 5/6] update --- python/tests/test_log_set_log_config.py | 38 +++++++++--------- python/utils/logger_file_headler.py | 53 ++++++++++++++----------- python/utils/logger_util.py | 53 ++++++++++++++----------- 3 files changed, 78 insertions(+), 66 deletions(-) diff --git a/python/tests/test_log_set_log_config.py b/python/tests/test_log_set_log_config.py index 155ded91b..e2e9a2a73 100644 --- a/python/tests/test_log_set_log_config.py +++ b/python/tests/test_log_set_log_config.py @@ -67,12 +67,12 @@ def test_rolling_with_time(self): logger = logging.getLogger('CustomLogger') logger.setLevel(logging.DEBUG) handler = SizeAndTimeRotatingFileHandler( - filename ='test.log', - when=time_msg, - interval=1, - max_bytes=1024*100*1024, - backup_count=10, - total_size_limit=1024*300*1024 + file_name ='test.log', + rotate_frequency=time_msg, + rotate_interval=1, + max_file_size=1024*100*1024, + max_log_files=10, + total_log_size=1024*300*1024 ) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) @@ -86,12 +86,12 @@ def test_rolling_with_size(self): logger = logging.getLogger('CustomLogger') logger.setLevel(logging.DEBUG) handler = SizeAndTimeRotatingFileHandler( - filename ='test.log', - when='S', - interval=10, - max_bytes=1*1024, - backup_count=2, - total_size_limit=1024*300*1024 + file_name ='test.log', + rotate_frequency='S', + rotate_interval=10, + max_file_size=1*1024, + max_log_files=2, + total_log_size=1024*300*1024 ) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) @@ -105,18 +105,18 @@ def test_rolling_to_total_max_size(self): logger = logging.getLogger('CustomLogger') logger.setLevel(logging.DEBUG) handler = SizeAndTimeRotatingFileHandler( - filename ='test.log', - when='S', - interval=100, - max_bytes=10*1024, - backup_count=10000, - total_size_limit=20*1024 + file_name ='test.log', + rotate_frequency='S', + rotate_interval=100, + max_file_size=10*1024, + max_log_files=10000, + total_log_size=20*1024 ) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) - for i in range(100): + for _ in range(100): logger.info("This is a test log message."*100) time.sleep(0.001) diff --git a/python/utils/logger_file_headler.py b/python/utils/logger_file_headler.py index dc310ce75..c27b97b58 100644 --- a/python/utils/logger_file_headler.py +++ b/python/utils/logger_file_headler.py @@ -18,22 +18,29 @@ from datetime import datetime, timedelta class SizeAndTimeRotatingFileHandler(logging.Handler): - def __init__(self, filename, when='S', interval=1, max_bytes=0, backup_count=0, total_size_limit=0): + def __init__(self, + file_name, + rotate_frequency='MIDNIGHT', + rotate_interval=1, + max_file_size=0, + max_log_files=0, + total_log_size=0 + ): super().__init__() - self.base_filename = filename - self.when = when.upper() - self.interval = interval - self.max_bytes = max_bytes - self.backup_count = backup_count - self.total_size_limit = total_size_limit + self.file_name = file_name + self.rotate_frequency = rotate_frequency.upper() + self.rotate_interval = rotate_interval + self.max_file_size = max_file_size + self.max_log_files = max_log_files + self.total_log_size = total_log_size self.current_time = datetime.now() - self.current_file = self.base_filename + self.current_file = self.file_name self.stream = open(self.current_file, 'a') self.last_rollover = time.time() def _get_new_filename(self): suffix = self.current_time.strftime("%Y-%m-%d_%H-%M-%S") - return f"{self.base_filename}.{suffix}" + return f"{self.file_name}.{suffix}" def emit(self, record): if self.shouldRollover(record): @@ -46,18 +53,18 @@ def shouldRollover(self, record): current_size = os.path.getsize(self.current_file) time_rollover = False - if self.when == 'S': - time_rollover = current_time >= self.last_rollover + self.interval - elif self.when == 'M': - time_rollover = current_time >= self.last_rollover + self.interval * 60 - elif self.when == 'H': - time_rollover = current_time >= self.last_rollover + self.interval * 3600 - elif self.when == 'D': - time_rollover = current_time >= self.last_rollover + self.interval * 86400 - elif self.when == 'MIDNIGHT': + if self.rotate_frequency == 'S': + time_rollover = current_time >= self.last_rollover + self.rotate_interval + elif self.rotate_frequency == 'M': + time_rollover = current_time >= self.last_rollover + self.rotate_interval * 60 + elif self.rotate_frequency == 'H': + time_rollover = current_time >= self.last_rollover + self.rotate_interval * 3600 + elif self.rotate_frequency == 'D': + time_rollover = current_time >= self.last_rollover + self.rotate_interval * 86400 + elif self.rotate_frequency == 'MIDNIGHT': time_rollover = datetime.fromtimestamp(current_time).date() != datetime.fromtimestamp(self.last_rollover).date() - size_rollover = current_size >= self.max_bytes if self.max_bytes > 0 else False + size_rollover = current_size >= self.max_file_size if self.max_file_size > 0 else False return time_rollover or size_rollover @@ -66,19 +73,19 @@ def doRollover(self): self.current_time = datetime.now() new_filename = self._get_new_filename() os.rename(self.current_file, new_filename) # Rename current file to new name - self.current_file = self.base_filename + self.current_file = self.file_name self.stream = open(self.current_file, 'a') self.last_rollover = time.time() self.manage_log_files() def manage_log_files(self): - log_files = sorted(glob.glob(f"{self.base_filename}.*"), key=os.path.getmtime) + log_files = sorted(glob.glob(f"{self.file_name}.*"), key=os.path.getmtime) - while len(log_files) > self.backup_count: + while len(log_files) > self.max_log_files: oldest_log = log_files.pop(0) os.remove(oldest_log) - while self._total_size(log_files) > self.total_size_limit: + while self._total_size(log_files) > self.total_log_size: if log_files: oldest_log = log_files.pop(0) os.remove(oldest_log) diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index bef94fe71..9bb07f504 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -68,24 +68,24 @@ 'level': 'ERROR', 'formatter': 'standard', 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', - 'filename': 'error.tmp.log', - 'when': 'MIDDNIGHT', - 'interval': 1, - 'max_bytes': 5*1024*1024, - 'backup_count': 20, - 'total_size_limit': 100*1024*1024 + 'file_name': 'error.tmp.log', + 'rotate_frequency': 'MIDDNIGHT', + 'rotate_interval': 1, + 'max_file_size': 5*1024*1024, + 'max_log_files': 20, + 'total_log_size': 100*1024*1024 } SET_CONFIG_HEADER = { 'level': 'DEBUG', 'formatter': 'standard', 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', - 'filename': 'tmp.log', - 'when': 'MIDNIGHT', - 'interval': 1, - 'max_bytes': 5*1024*1024, - 'backup_count': 20, - 'total_size_limit': 100*1024*1024 + 'file_name': 'tmp.log', + 'rotate_frequency': 'MIDDNIGHT', + 'rotate_interval': 1, + 'max_file_size': 5*1024*1024, + 'max_log_files': 20, + 'total_log_size': 100*1024*1024 } class LoggerWithLoggerId(logging.LoggerAdapter): @@ -99,6 +99,10 @@ def __init__(self, logger, extra, loglevel): LOGGING_CONFIG["handlers"] = {} LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] log_file = os.environ.get("APPBUILDER_LOGFILE", "") + loglevel = loglevel.strip().lower() + if loglevel not in ["debug", "info", "warning", "error"]: + raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % loglevel) + loglevel = loglevel.upper() if log_file: ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(log_file) FILE_HEADER["filename"] = log_file @@ -196,6 +200,7 @@ def setLogConfig(self, log_level = loglevel.strip().lower() if log_level not in ["debug", "info", "warning", "error"]: raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % log_level) + log_level = log_level.upper() # 设置console输出日志 if console_output: @@ -223,27 +228,27 @@ def setLogConfig(self, SET_CONFIG_HEADER['level'] = loglevel # 设置文件名称 - SET_CONFIG_HEADER['filename'] = file_name - ERROR_SET_CONFIG_HEADER['filename'] = _add_error_to_file_name(file_name) + SET_CONFIG_HEADER['file_name'] = file_name + ERROR_SET_CONFIG_HEADER['file_name'] = _add_error_to_file_name(file_name) # 设置滚动时间 - SET_CONFIG_HEADER['when'] = rotate_frequency - ERROR_SET_CONFIG_HEADER['when'] = rotate_frequency - SET_CONFIG_HEADER['interval'] = rotate_interval - ERROR_SET_CONFIG_HEADER['interval'] = rotate_interval + SET_CONFIG_HEADER['rotate_frequency'] = rotate_frequency + ERROR_SET_CONFIG_HEADER['rotate_frequency'] = rotate_frequency + SET_CONFIG_HEADER['rotate_interval'] = rotate_interval + ERROR_SET_CONFIG_HEADER['rotate_interval'] = rotate_interval # 设置最大文件大小 - SET_CONFIG_HEADER['max_bytes'] = max_file_size - ERROR_SET_CONFIG_HEADER['max_bytes'] = max_file_size + SET_CONFIG_HEADER['max_file_size'] = max_file_size + ERROR_SET_CONFIG_HEADER['max_file_size'] = max_file_size # 设置总大小限制 - SET_CONFIG_HEADER['total_size_limit'] = total_log_size - ERROR_SET_CONFIG_HEADER['total_size_limit'] = total_log_size + SET_CONFIG_HEADER['total_log_size'] = total_log_size + ERROR_SET_CONFIG_HEADER['total_log_size'] = total_log_size # 设置备份数量 - SET_CONFIG_HEADER['backup_count'] = max_log_files - ERROR_SET_CONFIG_HEADER['backup_count'] = max_log_files + SET_CONFIG_HEADER['max_log_files'] = max_log_files + ERROR_SET_CONFIG_HEADER['max_log_files'] = max_log_files LOGGING_CONFIG["handlers"]["file"] = SET_CONFIG_HEADER LOGGING_CONFIG["handlers"]["error_file"] = ERROR_SET_CONFIG_HEADER From d41647db68e12bc01e2333a066e10f5dc4da4f62 Mon Sep 17 00:00:00 2001 From: zhouhao10 Date: Thu, 2 Jan 2025 20:54:50 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=9A=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E6=97=A5=E5=BF=97=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/BasisModule/Trace/Debug.md | 5 +- python/tests/test_log_set_log_config.py | 20 ++++++- python/utils/logger_util.py | 73 +++++++++++++++++-------- 3 files changed, 71 insertions(+), 27 deletions(-) diff --git a/docs/BasisModule/Trace/Debug.md b/docs/BasisModule/Trace/Debug.md index a09fd0e91..f70baa3f1 100644 --- a/docs/BasisModule/Trace/Debug.md +++ b/docs/BasisModule/Trace/Debug.md @@ -60,7 +60,8 @@ Appbuilder-SDK新增滚动日志功能 主要参数: - console_output: 数据类型bool，默认值True，LOG日志是否在控制台输出 - loglevel: 数据类型str，默认值"DEBUG"，LOG日志级别 -- file_name: 数据类型str，默认值"tmp.log"，LOG日志名称 +- log_path: 数据类型str，默认值"/tmp"，默认日志存放路径。 +- file_name: 数据类型str，默认值为进程id，日志名前缀 - rotate_frequency: 数据类型str，默认值"MIDNIGHT"，LOG日志滚动更新时间单位 - "S": 以秒为单位 - "M": 以分钟为单位 @@ -78,7 +79,7 @@ Appbuilder-SDK新增滚动日志功能 appbuilder.logger.setLogConfig( console_output = False, loglevel="DEBUG" - file_name="appbuilder.log", + log_path="/tmp",, rotate_frequency="MIDNIGHT", # 每日凌晨更新 rotate_interval=1, max_file_size=100 * 1024 *1024, # 最大日志大小为100MB diff --git a/python/tests/test_log_set_log_config.py b/python/tests/test_log_set_log_config.py index e2e9a2a73..a4560b7b8 100644 --- a/python/tests/test_log_set_log_config.py +++ b/python/tests/test_log_set_log_config.py @@ -34,6 +34,21 @@ def test_set_log_config(self): max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize ) + def test_set_log_config_log_path(self): + os.environ["APPBUILDER_LOGPATH"] = "/tmp" + lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') + lwl.setLogConfig( + console_output = True, + loglevel='DEBUG', + log_path='/tmp', + file_name='test.log', + rotate_frequency='D', + rotate_interval=0, # 测试rotate_interval<1时，自动更新为1 + max_file_size=None, # 测试not max_file_size or max_file_size <= 0时，自动更新为sys.maxsize + total_log_size=None, # 测试not total_log_size or total_log_size <= 0时，自动更新为sys.maxsize + max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize + ) + def test_set_log_config_raise_error(self): lwl=LoggerWithLoggerId(logger='test_logger',extra={'logid':'test_logid'},loglevel='INFO') with self.assertRaises(ValueError): @@ -47,7 +62,7 @@ def test_set_log_config_raise_error(self): total_log_size=None, # 测试not total_log_size or total_log_size <= 0时，自动更新为sys.maxsize max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize ) - + with self.assertRaises(ValueError): lwl.setLogConfig( console_output = True, @@ -60,7 +75,6 @@ def test_set_log_config_raise_error(self): max_log_files=None, # 测试not max_log_files or max_log_files <= 0时，自动更新为sys.maxsize ) - def test_rolling_with_time(self): time_msgs = ['S', 'M', 'H', 'D', 'MIDNIGHT'] for time_msg in time_msgs: @@ -121,4 +135,4 @@ def test_rolling_to_total_max_size(self): time.sleep(0.001) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index 9bb07f504..fa25385e4 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -24,7 +24,7 @@ import logging.config from threading import current_thread from typing import Optional - + LOGGING_CONFIG = { "version": 1, "disable_existing_loggers": False, @@ -53,14 +53,14 @@ ERROR_FILE_HEADER = { "level": "ERROR", "class": "logging.FileHandler", - "filename": "error.tmp.log", + "filename": "tmp.error.log", "formatter": "standard", } FILE_HEADER = { "level": "DEBUG", "class": "logging.FileHandler", - "filename": "tmp.log", + "filename": "tmp.info.log", "formatter": "standard", } @@ -68,7 +68,7 @@ 'level': 'ERROR', 'formatter': 'standard', 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', - 'file_name': 'error.tmp.log', + 'file_name': 'tmp.error.log', 'rotate_frequency': 'MIDDNIGHT', 'rotate_interval': 1, 'max_file_size': 5*1024*1024, @@ -80,7 +80,7 @@ 'level': 'DEBUG', 'formatter': 'standard', 'class': 'appbuilder.SizeAndTimeRotatingFileHandler', - 'file_name': 'tmp.log', + 'file_name': 'tmp.info.log', 'rotate_frequency': 'MIDDNIGHT', 'rotate_interval': 1, 'max_file_size': 5*1024*1024, @@ -99,12 +99,29 @@ def __init__(self, logger, extra, loglevel): LOGGING_CONFIG["handlers"] = {} LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] log_file = os.environ.get("APPBUILDER_LOGFILE", "") + log_path = os.environ.get("APPBUILDER_LOGPATH", "") loglevel = loglevel.strip().lower() if loglevel not in ["debug", "info", "warning", "error"]: raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % loglevel) loglevel = loglevel.upper() - if log_file: - ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(log_file) + + if log_path: + current_pid = str(os.getpid()) + full_log_path = os.path.join(log_path, "log") + if not os.path.exists(full_log_path): + os.makedirs(full_log_path) + info_log_file = os.path.join(log_path, "log", current_pid + ".info.log") + error_log_file = os.path.join(log_path, "log", current_pid + ".error.log") + FILE_HEADER["filename"] = info_log_file + ERROR_FILE_HEADER["filename"] = error_log_file + FILE_HEADER["level"] = loglevel + LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") + if loglevel in ("DEBUG", "INFO", "WARNING"): + LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + elif log_file: + ERROR_FILE_HEADER["filename"] = self._add_error_to_file_name(log_file) FILE_HEADER["filename"] = log_file FILE_HEADER["level"] = loglevel LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER @@ -112,6 +129,7 @@ def __init__(self, logger, extra, loglevel): if loglevel in ("DEBUG", "INFO", "WARNING"): LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + CONSOLE_HEADER["level"] = loglevel LOGGING_CONFIG["handlers"]["console"] = CONSOLE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("console") @@ -148,6 +166,13 @@ def level(self): """ return self.logger.level + @staticmethod + def _add_error_to_file_name(filename): + prefix = "error." + dir_name, base_name = os.path.split(filename) + new_base_name = f"{prefix}{base_name}" + return os.path.join(dir_name, new_base_name) + def setFilename(self, filename): """ set filename @@ -157,11 +182,11 @@ def setFilename(self, filename): LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: - ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(filename) + ERROR_FILE_HEADER["filename"] = self._add_error_to_file_name(filename) LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") FILE_HEADER["filename"] = filename - ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(filename) + ERROR_FILE_HEADER["filename"] = self._add_error_to_file_name(filename) LOGGING_CONFIG["handlers"]["file"] = FILE_HEADER LOGGING_CONFIG["handlers"]["error_file"] = ERROR_FILE_HEADER logging.config.dictConfig(LOGGING_CONFIG) @@ -186,16 +211,17 @@ def setLoglevel(self, level): def setLogConfig(self, console_output: bool = True, loglevel: str = "DEBUG", - file_name: str = "tmp.log", + log_path: str = "/tmp", rotate_frequency: str = "MIDNIGHT", rotate_interval: int = 1, max_file_size: Optional[int] = None, # 以B为单位 total_log_size: Optional[int] = None, # 以B为单位 - max_log_files: Optional[int] = None + max_log_files: Optional[int] = None, + file_name: Optional[str] = None ): LOGGING_CONFIG["handlers"] = {} LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = [] - + # log_level 数据校验 log_level = loglevel.strip().lower() if log_level not in ["debug", "info", "warning", "error"]: @@ -209,7 +235,7 @@ def setLogConfig(self, LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("console") else: LOGGING_CONFIG["loggers"]["appbuilder"]["propagate"] = False - + # 参数验证 if not max_file_size or max_file_size <= 0: max_file_size = sys.maxsize @@ -228,8 +254,17 @@ def setLogConfig(self, SET_CONFIG_HEADER['level'] = loglevel # 设置文件名称 - SET_CONFIG_HEADER['file_name'] = file_name - ERROR_SET_CONFIG_HEADER['file_name'] = _add_error_to_file_name(file_name) + if not file_name: + current_pid = str(os.getpid()) + else: + current_pid = file_name + full_log_path = os.path.join(log_path, "log") + if not os.path.exists(full_log_path): + os.makedirs(full_log_path) + info_log_file = os.path.join(log_path, "log", current_pid + ".info.log") + error_log_file = os.path.join(log_path, "log", current_pid + ".error.log") + SET_CONFIG_HEADER["file_name"] = info_log_file + ERROR_SET_CONFIG_HEADER["file_name"] = error_log_file # 设置滚动时间 SET_CONFIG_HEADER['rotate_frequency'] = rotate_frequency @@ -238,7 +273,7 @@ def setLogConfig(self, ERROR_SET_CONFIG_HEADER['rotate_interval'] = rotate_interval # 设置最大文件大小 - + SET_CONFIG_HEADER['max_file_size'] = max_file_size ERROR_SET_CONFIG_HEADER['max_file_size'] = max_file_size @@ -320,11 +355,5 @@ def get_logger(name, level=logging.INFO): logger.propagate = False return logger -def _add_error_to_file_name(filename): - prefix = "error." - dir_name, base_name = os.path.split(filename) - new_base_name = f"{prefix}{base_name}" - return os.path.join(dir_name, new_base_name) - logger = _setup_logging()