Files
douyi_guaji_main/douyin/logging/douyin_logging_pkg/config.py
T

654 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""douyin_logging_pkg.config - 日志配置与工具。
- get_logger 带缓存的 logger 获取
- configure_logging 主配置入口(应用 handlers + cleanup + monitor)
- log_function_call 性能装饰器:记录函数调用耗时与异常
- LogContext 上下文管理器:记录代码块执行时间
- get_caller_info 栈帧信息抓取
- format_exception_info 异常堆栈格式化
"""
from __future__ import annotations
import glob
import inspect
import logging
import logging.handlers
import os
import sys
import threading
import time
import traceback
from datetime import datetime
from functools import wraps
from typing import Any, Callable, Dict, Optional
from douyin.core import path_config
from . import _common as _state
from .handlers import DatedFileHandler, RotatingDatedFileHandler
from .cleanup import (
_load_logging_cleanup_policy_from_config,
cleanup_old_logs,
compress_log_file,
)
from .monitor import start_log_monitor
def get_logger(name: Optional[str] = None) -> logging.Logger:
"""获取指定名称的 logger,如果未配置则自动配置"""
if not _state._logging_configured:
configure_logging()
# Ensure name is never None
if name is None:
# 自动获取调用者模块名
frame = None
try:
frame = inspect.currentframe()
if frame and frame.f_back:
caller_frame = frame.f_back
name = caller_frame.f_globals.get("__name__", "root")
else:
name = "root"
except Exception:
name = "root"
finally:
if frame is not None:
try:
del frame
except Exception:
pass
# Final fallback to ensure name is a string
logger_name: str = name if name is not None else "root"
if logger_name not in _state._logger_cache:
_state._logger_cache[logger_name] = logging.getLogger(logger_name)
return _state._logger_cache[logger_name]
def configure_logging(
log_level: Optional[str] = None,
log_file: Optional[str] = None,
max_bytes: int = 10 * 1024 * 1024, # 10MB
backup_count: int = 5, # 同一日期内保留的文件数
enable_console: bool = True,
enable_file: bool = True,
keep_days: Optional[int] = None, # 保留日志天数
use_dated_logs: Optional[bool] = None, # 是否使用按日期分割的日志
compress_logs: Optional[bool] = None, # 是否压缩旧日志
max_total_size_gb: float = 10.0, # 最大总日志大小(GB)
compress_after_days: int = 7, # 几天后压缩
enable_monitor: Optional[bool] = None, # 是否启用日志监控(仅启动阶段检查;运行时监控需显式开启)
monitor_warning_mb: Optional[float] = None, # 监控警告大小(MB)
monitor_critical_mb: Optional[float] = None, # 监控严重大小(MB)
) -> None:
"""配置日志系统
参数:
log_level: 日志级别 (DEBUG, INFO, WARNING, ERROR, CRITICAL),默认从 config.json 读取
log_file: 日志文件基础路径,默认使用 logs/unified.log(会被转换为 logs/unified)
max_bytes: 单个日志文件最大大小(字节),超过后轮转,默认100MB
backup_count: 保留的备份文件数量,默认10
enable_console: 是否输出到控制台
enable_file: 是否输出到文件
keep_days: 保留日志的天数,超过此天数的日志将被自动删除,默认30天
use_dated_logs: 是否使用按日期分割的日志,默认True
compress_logs: 是否压缩旧日志,默认True
max_total_size_gb: 最大总日志大小(GB),超过后删除最旧文件,默认10GB
compress_after_days: 几天后压缩日志,默认7天
enable_monitor: 是否启用日志监控(默认从 config.json 读取;仅影响启动阶段检查)
monitor_warning_mb: 监控警告大小阈值(MB),默认500MB
monitor_critical_mb: 监控严重大小阈值(MB),默认2000MB
"""
# rotation 相关默认值(函数级),确保在所有分支可用,避免可能的未赋值使用
rotation_enabled = False
rotation_when = None
rotation_interval = 1
# 从 config.json 读取配置
if (
log_level is None
or log_file is None
or keep_days is None
or use_dated_logs is None
):
try:
import json
config_path = path_config.get_config_path()
if os.path.exists(config_path):
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
if log_level is None:
log_level = config.get("log_level", "INFO")
if log_file is None:
# 优先读取 log_unified_path,其次读取 logging.file_path
log_file = config.get("log_unified_path") or config.get(
"logging", {}
).get("file_path", "logs/unified.log")
log_config = config.get("logging", {})
if keep_days is None:
keep_days = log_config.get("keep_days", 30)
if use_dated_logs is None:
use_dated_logs = log_config.get("use_dated_logs", True)
if compress_logs is None:
compress_logs = log_config.get("compress_logs", True)
if max_total_size_gb == 10.0: # 如果是默认值,从配置读取
max_total_size_gb = log_config.get("max_total_size_gb", 10.0)
if compress_after_days == 7: # 如果是默认值,从配置读取
compress_after_days = log_config.get("compress_after_days", 7)
# 读取 monitor 配置
if enable_monitor is None:
enable_monitor = bool(log_config.get("enable_monitor", False))
if monitor_warning_mb is None:
try:
monitor_warning_mb = float(log_config.get("monitor_warning_mb", 500.0))
except Exception:
monitor_warning_mb = 500.0
if monitor_critical_mb is None:
try:
monitor_critical_mb = float(log_config.get("monitor_critical_mb", 2000.0))
except Exception:
monitor_critical_mb = 2000.0
# 读取 rotation 配置(优先级高于函数默认参数)
rotation_cfg = (
log_config.get("rotation", {})
if isinstance(log_config, dict)
else {}
)
rotation_enabled = rotation_cfg.get("enabled", True)
if rotation_cfg:
# 支持按大小与按时间的配置
max_bytes = rotation_cfg.get("max_bytes", max_bytes)
backup_count = rotation_cfg.get("backup_count", backup_count)
rotation_when = rotation_cfg.get("when")
rotation_interval = rotation_cfg.get("interval", 1)
else:
rotation_when = None
rotation_interval = 1
except Exception:
if log_level is None:
log_level = "INFO"
if log_file is None:
log_file = "logs/unified.log"
if keep_days is None:
keep_days = 30
if use_dated_logs is None:
use_dated_logs = True
if compress_logs is None:
compress_logs = True
if enable_monitor is None:
enable_monitor = False
if monitor_warning_mb is None:
monitor_warning_mb = 500.0
if monitor_critical_mb is None:
monitor_critical_mb = 2000.0
# rotation 默认值(当读取 config 出错时)
rotation_when = None
rotation_interval = 1
rotation_enabled = False
# 规范化 monitor 默认值
if enable_monitor is None:
enable_monitor = False
if monitor_warning_mb is None:
monitor_warning_mb = 500.0
if monitor_critical_mb is None:
monitor_critical_mb = 2000.0
# 规范化日志级别
if log_level is None:
log_level = "INFO"
numeric_level = getattr(logging, log_level.upper(), logging.INFO)
# 配置根 logger
root_logger = logging.getLogger()
root_logger.setLevel(numeric_level)
# 清除已有的 handlers(避免重复配置)
root_logger.handlers.clear()
# 简化的日志格式:时间戳 - [级别] - [模块] - 消息
# 去掉了冗余的线程名、函数名、行号、traceId等信息,使日志更简洁
detailed_format = "%(asctime)s - [%(levelname)s] - [%(name)s] - %(message)s"
simple_format = "%(asctime)s - [%(levelname)s] - %(message)s"
# 控制台输出
if enable_console:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(numeric_level)
console_formatter = logging.Formatter(
simple_format, datefmt="%Y-%m-%d %H:%M:%S"
)
console_handler.setFormatter(console_formatter)
class ConsoleSuppressFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
return not getattr(record, "suppress_console_output", False)
console_handler.addFilter(ConsoleSuppressFilter())
root_logger.addHandler(console_handler)
# 文件输出(按日期分割或传统轮转)
if enable_file:
try:
# 确保日志目录存在
if log_file is None:
log_file = "logs/unified.log" # 默认值
if not os.path.isabs(log_file):
log_file = os.path.join(path_config.PROJECT_ROOT, log_file)
# 如果是使用日期日志,提取基础文件名(移除扩展名)
if use_dated_logs:
# 从完整路径提取基础名称,例如: logs/unified.log -> logs/unified
if log_file:
base_filename = log_file.replace(".log", "")
else:
base_filename = "logs/unified"
log_dir = os.path.dirname(base_filename)
# 确保目录存在
if log_dir:
os.makedirs(log_dir, exist_ok=True)
# 使用增强的轮转Handler
# 支持每次运行生成独立文件(per_run):在 base_filename 后追加运行时间戳,并在 handler 中关闭日期轮转
per_run = False
try:
per_run = (
bool(log_config.get("per_run", True))
if isinstance(log_config, dict)
else True
)
except Exception:
per_run = True
# base_filename_base: 不带运行时间戳/扩展名的基础名,用于清理和监控
base_filename_base = base_filename
if per_run:
run_ts = datetime.now().strftime("%Y-%m-%d_%H%M%S")
base_filename = f"{base_filename_base}_{run_ts}"
file_handler = RotatingDatedFileHandler(
base_filename,
date_format="%Y-%m-%d",
max_bytes=max_bytes,
backup_count=backup_count,
encoding="utf-8-sig",
mode="a",
errors="replace",
compress_old=compress_logs if compress_logs is not None else True,
per_run=per_run,
)
# 启动时检查总日志大小:若超过警告阈值则同步清理一次(仅在启动执行)
if keep_days is not None:
try:
# 计算当前日志总大小(MB)
pattern = os.path.join(
log_dir or path_config.get_logs_dir(),
f"{os.path.basename(base_filename_base)}_*.log*",
)
total_size = 0
for p in glob.glob(pattern):
try:
if os.path.exists(p):
total_size += os.path.getsize(p)
except Exception:
pass
total_size_mb = total_size / 1024.0 / 1024.0
except Exception:
total_size_mb = 0.0
try:
if total_size_mb >= float(monitor_warning_mb or 500.0):
# 同步清理一次(仅在启动阶段)
cleanup_old_logs(
log_dir or path_config.get_logs_dir(),
os.path.basename(base_filename_base),
keep_days=keep_days,
max_total_size_gb=max_total_size_gb,
compress_after_days=compress_after_days,
compress=(
compress_logs if compress_logs is not None else True
),
)
except Exception:
pass
# 注意:按用户反馈,默认不在运行过程中做日志目录体积检查。
# 仅当显式开启运行时监控时才启动后台线程(避免主流程中途 I/O 干扰与刷屏)。
runtime_monitor_enabled = False
try:
runtime_monitor_enabled = bool(
(log_config.get("enable_runtime_monitor", False) if isinstance(log_config, dict) else False)
)
except Exception:
runtime_monitor_enabled = False
# 环境变量可强制开启(便于临时诊断)
if os.environ.get("ENABLE_RUNTIME_LOG_MONITOR") == "1":
runtime_monitor_enabled = True
if runtime_monitor_enabled:
try:
monitor_check_interval = 300
try:
if isinstance(log_config, dict):
monitor_check_interval = int(
log_config.get("monitor_check_interval", 300)
)
except Exception:
monitor_check_interval = 300
start_log_monitor(
logs_dir=log_dir or path_config.get_logs_dir(),
base_name=os.path.basename(base_filename_base),
warning_size_mb=float(monitor_warning_mb),
critical_size_mb=float(monitor_critical_mb),
check_interval=monitor_check_interval,
enable_runtime_cleanup=bool(
(log_config.get("enable_runtime_cleanup", False) if isinstance(log_config, dict) else False)
),
)
except Exception as e:
try:
print(f"[日志配置] 启动运行时监控器失败: {e}")
except Exception:
pass
else:
# 使用传统的RotatingFileHandler
if log_file is None:
log_file = "logs/unified.log" # 默认值
log_dir = os.path.dirname(log_file)
if log_dir:
os.makedirs(log_dir, exist_ok=True)
# 如果配置了按时间轮转(when),使用 TimedRotatingFileHandler
try:
if rotation_enabled and rotation_when:
file_handler = logging.handlers.TimedRotatingFileHandler(
log_file,
when=rotation_when,
interval=rotation_interval,
backupCount=backup_count,
encoding="utf-8-sig",
errors="replace",
)
else:
file_handler = logging.handlers.RotatingFileHandler(
log_file, # type: ignore[arg-type]
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8-sig",
errors="replace",
)
except Exception:
# 回退到大小轮转
file_handler = logging.handlers.RotatingFileHandler(
log_file, # type: ignore[arg-type]
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8-sig",
errors="replace",
)
file_handler.setLevel(numeric_level)
# 使用自定义Formatter以支持traceId
try:
from douyin.logging.unified_logger import UnifiedFormatter
file_formatter = UnifiedFormatter(
detailed_format, datefmt="%Y-%m-%d %H:%M:%S"
)
except ImportError:
# 如果unified_logger不可用,使用标准Formatter
file_formatter = logging.Formatter(
detailed_format, datefmt="%Y-%m-%d %H:%M:%S"
)
# 添加traceId占位符处理
class TraceIdFormatter(logging.Formatter):
def format(self, record):
if not hasattr(record, "traceId"):
record.traceId = "N/A"
return super().format(record)
file_formatter = TraceIdFormatter(
detailed_format, datefmt="%Y-%m-%d %H:%M:%S"
)
file_formatter.default_msec_format = "%s.%03d" # 毫秒精度
file_handler.setFormatter(file_formatter)
class FileSuppressFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
return not getattr(record, "suppress_file_output", False)
file_handler.addFilter(FileSuppressFilter())
root_logger.addHandler(file_handler)
except Exception as e:
# 文件 handler 创建失败,只输出到控制台
try:
print(f"[日志配置] 警告:无法创建日志文件 {log_file}: {e}")
except Exception:
pass
_state._logging_configured = True
def log_function_call(
logger: Optional[logging.Logger] = None,
log_args: bool = True,
log_result: bool = False,
log_time: bool = True,
min_duration: float = 0.0, # 只记录超过此耗时的调用
):
"""装饰器:记录函数调用信息(参数、返回值、耗时)
参数:
logger: 使用的 logger,默认自动获取
log_args: 是否记录函数参数
log_result: 是否记录返回值
log_time: 是否记录执行时间
min_duration: 最小耗时阈值(秒),低于此值不记录耗时
"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs):
if logger is None:
_logger = get_logger()
else:
_logger = logger
func_name = func.__name__
start_time = time.time()
# 记录函数调用
if log_args:
args_str = ", ".join(
[str(arg)[:100] for arg in args[:3]]
) # 只记录前3个参数,避免过长
kwargs_str = ", ".join(
[f"{k}={str(v)[:50]}" for k, v in list(kwargs.items())[:3]]
)
call_info = (
f"{func_name}({args_str}{', ' + kwargs_str if kwargs_str else ''})"
)
if len(args) > 3 or len(kwargs) > 3:
call_info += "..."
_logger.debug(f"调用函数: {call_info}")
# 执行函数
try:
result = func(*args, **kwargs)
elapsed = time.time() - start_time
# 记录执行结果
if log_time and elapsed >= min_duration:
_logger.debug(f"函数 {func_name} 执行完成,耗时: {elapsed:.3f}秒")
if log_result:
result_str = str(result)[:200] # 限制长度
_logger.debug(f"函数 {func_name} 返回值: {result_str}")
return result
except Exception as e:
elapsed = time.time() - start_time
_logger.exception(
f"函数 {func_name} 执行异常,耗时: {elapsed:.3f}秒", exc_info=True
)
raise
return wrapper
return decorator
class LogContext:
"""上下文管理器:记录代码块执行时间"""
def __init__(
self,
message: str,
logger: Optional[logging.Logger] = None,
level: str = "DEBUG",
log_on_exit: bool = True,
):
self.message = message
self.logger = logger or get_logger()
self.level = getattr(logging, level.upper(), logging.DEBUG)
self.log_on_exit = log_on_exit
self.start_time: Optional[float] = None
def __enter__(self):
self.start_time = time.time()
self.logger.log(self.level, f"{self.message} [开始]")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if not self.log_on_exit:
return False
if self.start_time is None:
elapsed = 0.0
else:
elapsed = time.time() - self.start_time
if exc_type is None:
self.logger.log(self.level, f"{self.message} [完成] 耗时: {elapsed:.3f}秒")
else:
self.logger.log(
logging.ERROR,
f"{self.message} [异常] 耗时: {elapsed:.3f}秒",
exc_info=(exc_type, exc_val, exc_tb),
)
return False # 不抑制异常
def get_caller_info(skip_frames: int = 0) -> Dict[str, Any]:
"""获取调用者信息(模块名、函数名、行号)
参数:
skip_frames: 跳过的栈帧数(用于在装饰器中获取真正的调用者)
返回:
包含 module, function, line, file 的字典
"""
frame = None
try:
frame = inspect.currentframe()
if frame:
# 跳过当前函数和装饰器
for _ in range(skip_frames + 2):
if frame.f_back:
frame = frame.f_back
else:
break
if frame:
return {
"module": os.path.basename(frame.f_code.co_filename),
"function": frame.f_code.co_name,
"line": frame.f_lineno,
"file": frame.f_code.co_filename,
}
except Exception:
pass
finally:
if frame is not None:
try:
del frame
except Exception:
pass
return {
"module": "unknown",
"function": "unknown",
"line": 0,
"file": "unknown",
}
def format_exception_info(exc_info=None) -> str:
"""格式化异常信息为字符串
参数:
exc_info: 异常信息元组 (type, value, traceback),默认使用 sys.exc_info()
返回:
格式化的异常堆栈字符串
"""
if exc_info is None:
exc_info = sys.exc_info()
if exc_info[0] is None:
return ""
try:
# exc_info 应该是 (type, value, traceback) 元组
if isinstance(exc_info, tuple) and len(exc_info) >= 3:
exc_type, exc_value, exc_tb = exc_info[0], exc_info[1], exc_info[2]
# 确保类型正确
if isinstance(exc_value, BaseException):
return "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
else:
# 如果不是异常实例,创建一个包装
return "".join(
traceback.format_exception(
exc_type,
(
BaseException(str(exc_value))
if exc_value
else BaseException("Unknown error")
),
exc_tb,
)
)
else:
return "".join(traceback.format_exception(*exc_info))
except Exception:
return (
f"异常: {exc_info[1] if len(exc_info) > 1 and exc_info[1] else 'Unknown'}"
)