Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
yinjiaqi authored and yinjiaqi committed Jan 2, 2025
1 parent b2d2df1 commit 6fb10ec
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 66 deletions.
38 changes: 19 additions & 19 deletions python/tests/test_log_set_log_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,12 @@ def test_rolling_with_time(self):
logger = logging.getLogger('CustomLogger')
logger.setLevel(logging.DEBUG)
handler = SizeAndTimeRotatingFileHandler(
filename ='test.log',
when=time_msg,
interval=1,
max_bytes=1024*100*1024,
backup_count=10,
total_size_limit=1024*300*1024
file_name ='test.log',
rotate_frequency=time_msg,
rotate_interval=1,
max_file_size=1024*100*1024,
max_log_files=10,
total_log_size=1024*300*1024
)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
Expand All @@ -86,12 +86,12 @@ def test_rolling_with_size(self):
logger = logging.getLogger('CustomLogger')
logger.setLevel(logging.DEBUG)
handler = SizeAndTimeRotatingFileHandler(
filename ='test.log',
when='S',
interval=10,
max_bytes=1*1024,
backup_count=2,
total_size_limit=1024*300*1024
file_name ='test.log',
rotate_frequency='S',
rotate_interval=10,
max_file_size=1*1024,
max_log_files=2,
total_log_size=1024*300*1024
)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
Expand All @@ -105,18 +105,18 @@ def test_rolling_to_total_max_size(self):
logger = logging.getLogger('CustomLogger')
logger.setLevel(logging.DEBUG)
handler = SizeAndTimeRotatingFileHandler(
filename ='test.log',
when='S',
interval=100,
max_bytes=10*1024,
backup_count=10000,
total_size_limit=20*1024
file_name ='test.log',
rotate_frequency='S',
rotate_interval=100,
max_file_size=10*1024,
max_log_files=10000,
total_log_size=20*1024
)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

for i in range(100):
for _ in range(100):
logger.info("This is a test log message."*100)
time.sleep(0.001)

Expand Down
53 changes: 30 additions & 23 deletions python/utils/logger_file_headler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,29 @@
from datetime import datetime, timedelta

class SizeAndTimeRotatingFileHandler(logging.Handler):
def __init__(self, filename, when='S', interval=1, max_bytes=0, backup_count=0, total_size_limit=0):
def __init__(self,
file_name,
rotate_frequency='MIDNIGHT',
rotate_interval=1,
max_file_size=0,
max_log_files=0,
total_log_size=0
):
super().__init__()
self.base_filename = filename
self.when = when.upper()
self.interval = interval
self.max_bytes = max_bytes
self.backup_count = backup_count
self.total_size_limit = total_size_limit
self.file_name = file_name
self.rotate_frequency = rotate_frequency.upper()
self.rotate_interval = rotate_interval
self.max_file_size = max_file_size
self.max_log_files = max_log_files
self.total_log_size = total_log_size
self.current_time = datetime.now()
self.current_file = self.base_filename
self.current_file = self.file_name
self.stream = open(self.current_file, 'a')
self.last_rollover = time.time()

def _get_new_filename(self):
suffix = self.current_time.strftime("%Y-%m-%d_%H-%M-%S")
return f"{self.base_filename}.{suffix}"
return f"{self.file_name}.{suffix}"

def emit(self, record):
if self.shouldRollover(record):
Expand All @@ -46,18 +53,18 @@ def shouldRollover(self, record):
current_size = os.path.getsize(self.current_file)

time_rollover = False
if self.when == 'S':
time_rollover = current_time >= self.last_rollover + self.interval
elif self.when == 'M':
time_rollover = current_time >= self.last_rollover + self.interval * 60
elif self.when == 'H':
time_rollover = current_time >= self.last_rollover + self.interval * 3600
elif self.when == 'D':
time_rollover = current_time >= self.last_rollover + self.interval * 86400
elif self.when == 'MIDNIGHT':
if self.rotate_frequency == 'S':
time_rollover = current_time >= self.last_rollover + self.rotate_interval
elif self.rotate_frequency == 'M':
time_rollover = current_time >= self.last_rollover + self.rotate_interval * 60
elif self.rotate_frequency == 'H':
time_rollover = current_time >= self.last_rollover + self.rotate_interval * 3600
elif self.rotate_frequency == 'D':
time_rollover = current_time >= self.last_rollover + self.rotate_interval * 86400
elif self.rotate_frequency == 'MIDNIGHT':
time_rollover = datetime.fromtimestamp(current_time).date() != datetime.fromtimestamp(self.last_rollover).date()

size_rollover = current_size >= self.max_bytes if self.max_bytes > 0 else False
size_rollover = current_size >= self.max_file_size if self.max_file_size > 0 else False

return time_rollover or size_rollover

Expand All @@ -66,19 +73,19 @@ def doRollover(self):
self.current_time = datetime.now()
new_filename = self._get_new_filename()
os.rename(self.current_file, new_filename) # Rename current file to new name
self.current_file = self.base_filename
self.current_file = self.file_name
self.stream = open(self.current_file, 'a')
self.last_rollover = time.time()
self.manage_log_files()

def manage_log_files(self):
log_files = sorted(glob.glob(f"{self.base_filename}.*"), key=os.path.getmtime)
log_files = sorted(glob.glob(f"{self.file_name}.*"), key=os.path.getmtime)

while len(log_files) > self.backup_count:
while len(log_files) > self.max_log_files:
oldest_log = log_files.pop(0)
os.remove(oldest_log)

while self._total_size(log_files) > self.total_size_limit:
while self._total_size(log_files) > self.total_log_size:
if log_files:
oldest_log = log_files.pop(0)
os.remove(oldest_log)
Expand Down
53 changes: 29 additions & 24 deletions python/utils/logger_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,24 +68,24 @@
'level': 'ERROR',
'formatter': 'standard',
'class': 'appbuilder.SizeAndTimeRotatingFileHandler',
'filename': 'error.tmp.log',
'when': 'MIDDNIGHT',
'interval': 1,
'max_bytes': 5*1024*1024,
'backup_count': 20,
'total_size_limit': 100*1024*1024
'file_name': 'error.tmp.log',
'rotate_frequency': 'MIDDNIGHT',
'rotate_interval': 1,
'max_file_size': 5*1024*1024,
'max_log_files': 20,
'total_log_size': 100*1024*1024
}

SET_CONFIG_HEADER = {
'level': 'DEBUG',
'formatter': 'standard',
'class': 'appbuilder.SizeAndTimeRotatingFileHandler',
'filename': 'tmp.log',
'when': 'MIDNIGHT',
'interval': 1,
'max_bytes': 5*1024*1024,
'backup_count': 20,
'total_size_limit': 100*1024*1024
'file_name': 'tmp.log',
'rotate_frequency': 'MIDDNIGHT',
'rotate_interval': 1,
'max_file_size': 5*1024*1024,
'max_log_files': 20,
'total_log_size': 100*1024*1024
}

class LoggerWithLoggerId(logging.LoggerAdapter):
Expand All @@ -99,6 +99,10 @@ def __init__(self, logger, extra, loglevel):
LOGGING_CONFIG["handlers"] = {}
LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"] = []
log_file = os.environ.get("APPBUILDER_LOGFILE", "")
loglevel = loglevel.strip().lower()
if loglevel not in ["debug", "info", "warning", "error"]:
raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % loglevel)
loglevel = loglevel.upper()
if log_file:
ERROR_FILE_HEADER["filename"] = _add_error_to_file_name(log_file)
FILE_HEADER["filename"] = log_file
Expand Down Expand Up @@ -196,6 +200,7 @@ def setLogConfig(self,
log_level = loglevel.strip().lower()
if log_level not in ["debug", "info", "warning", "error"]:
raise ValueError("expected APPBUILDER_LOGLEVEL in [debug, info, warning, error], but got %s" % log_level)
log_level = log_level.upper()

# 设置console输出日志
if console_output:
Expand Down Expand Up @@ -223,27 +228,27 @@ def setLogConfig(self,
SET_CONFIG_HEADER['level'] = loglevel

# 设置文件名称
SET_CONFIG_HEADER['filename'] = file_name
ERROR_SET_CONFIG_HEADER['filename'] = _add_error_to_file_name(file_name)
SET_CONFIG_HEADER['file_name'] = file_name
ERROR_SET_CONFIG_HEADER['file_name'] = _add_error_to_file_name(file_name)

# 设置滚动时间
SET_CONFIG_HEADER['when'] = rotate_frequency
ERROR_SET_CONFIG_HEADER['when'] = rotate_frequency
SET_CONFIG_HEADER['interval'] = rotate_interval
ERROR_SET_CONFIG_HEADER['interval'] = rotate_interval
SET_CONFIG_HEADER['rotate_frequency'] = rotate_frequency
ERROR_SET_CONFIG_HEADER['rotate_frequency'] = rotate_frequency
SET_CONFIG_HEADER['rotate_interval'] = rotate_interval
ERROR_SET_CONFIG_HEADER['rotate_interval'] = rotate_interval

# 设置最大文件大小

SET_CONFIG_HEADER['max_bytes'] = max_file_size
ERROR_SET_CONFIG_HEADER['max_bytes'] = max_file_size
SET_CONFIG_HEADER['max_file_size'] = max_file_size
ERROR_SET_CONFIG_HEADER['max_file_size'] = max_file_size

# 设置总大小限制
SET_CONFIG_HEADER['total_size_limit'] = total_log_size
ERROR_SET_CONFIG_HEADER['total_size_limit'] = total_log_size
SET_CONFIG_HEADER['total_log_size'] = total_log_size
ERROR_SET_CONFIG_HEADER['total_log_size'] = total_log_size

# 设置备份数量
SET_CONFIG_HEADER['backup_count'] = max_log_files
ERROR_SET_CONFIG_HEADER['backup_count'] = max_log_files
SET_CONFIG_HEADER['max_log_files'] = max_log_files
ERROR_SET_CONFIG_HEADER['max_log_files'] = max_log_files

LOGGING_CONFIG["handlers"]["file"] = SET_CONFIG_HEADER
LOGGING_CONFIG["handlers"]["error_file"] = ERROR_SET_CONFIG_HEADER
Expand Down

0 comments on commit 6fb10ec

Please sign in to comment.