Files
douyin_guaji-main/douyin_logging.py
T
Your Name e68f1dd3de feat: Refactor coverage management and enhance logging functionality
- Removed the `.coverage` file from the repository to streamline coverage management.
- Updated `.gitignore` to exclude coverage-related files and directories, ensuring a cleaner project structure.
- Introduced new scripts for checking GPU availability and detecting GUI parameter conflicts, enhancing the overall functionality of the application.
- Improved logging mechanisms across various modules to support better debugging and monitoring.
- Added comprehensive documentation for new features and updated existing markdown files for clarity.
2025-12-23 02:38:51 +08:00

584 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
日志系统配置与工具模块
====================
功能:
1. 标准 logging 配置(支持日志轮转、级别管理)
2. 按日期分割日志(每天一个新日志文件)
3. 自动删除过旧日志(可配置保留天数)
4. 增强的日志记录工具(自动记录调用者信息、异常堆栈等)
5. 性能监控装饰器(记录函数调用耗时)
6. 上下文管理器(记录代码块执行时间)
"""
from __future__ import annotations
import os
import sys
import time
import logging
import logging.handlers
import traceback
import inspect
import glob
from datetime import datetime, timedelta
from typing import Optional, Callable, Any, Dict
from functools import wraps
import path_config
# 全局日志配置
_logging_configured = False
_logger_cache: Dict[str, logging.Logger] = {}
class DatedFileHandler(logging.FileHandler):
"""按日期创建日志文件的Handler,每天创建一个新文件"""
def __init__(
self,
base_filename: str,
date_format: str = "%Y-%m-%d",
encoding: str = "utf-8-sig",
mode: str = "a",
errors: str = "replace",
):
"""
参数:
base_filename: 基础文件名(不含日期和扩展名),如 "logs/unified"
date_format: 日期格式,默认 "%Y-%m-%d" (例如: 2024-01-15)
encoding: 文件编码,默认 "utf-8-sig" (带BOM,避免Windows乱码)
mode: 打开模式,默认 "a" (追加)
errors: 错误处理,默认 "replace"
"""
self.base_filename = base_filename
self.date_format = date_format
self.current_date = datetime.now().strftime(date_format)
self.current_filename = f"{base_filename}_{self.current_date}.log"
# 确保目录存在
log_dir = os.path.dirname(self.current_filename)
if log_dir:
os.makedirs(log_dir, exist_ok=True)
super().__init__(
self.current_filename,
mode=mode,
encoding=encoding,
errors=errors,
)
def emit(self, record: logging.LogRecord) -> None:
"""发送日志记录,如果日期变化则切换到新文件(性能优化版本)"""
# 优化:缓存日期检查,避免每次调用都执行 datetime.now().strftime()
# 使用记录计数器和时间戳来减少日期检查频率
if not hasattr(self, '_date_check_counter'):
self._date_check_counter = 0
self._last_date_check_time = datetime.now()
# 每100条记录或每1秒检查一次日期(平衡性能和准确性)
self._date_check_counter += 1
should_check_date = (
self._date_check_counter >= 100 or
(datetime.now() - self._last_date_check_time).total_seconds() >= 1.0
)
if should_check_date:
today = datetime.now().strftime(self.date_format)
self._last_date_check_time = datetime.now()
if today != self.current_date:
# 日期变化,关闭旧文件,打开新文件
self.close()
self.current_date = today
self.current_filename = f"{self.base_filename}_{self.current_date}.log"
# 确保目录存在
log_dir = os.path.dirname(self.current_filename)
if log_dir:
os.makedirs(log_dir, exist_ok=True)
# 重新初始化文件流
self.baseFilename = self.current_filename
self.stream = self._open()
# 重置计数器
self._date_check_counter = 0
super().emit(record)
def cleanup_old_logs(logs_dir: str, base_name: str, keep_days: int = 30) -> None:
"""删除过旧的日志文件
参数:
logs_dir: 日志目录路径
base_name: 日志文件基础名称(不含日期和扩展名),如 "unified"
keep_days: 保留天数,默认30天,超过此天数的日志将被删除
"""
try:
if not os.path.exists(logs_dir):
return
# 构建匹配模式,例如: logs/unified_*.log
pattern = os.path.join(logs_dir, f"{base_name}_*.log")
log_files = glob.glob(pattern)
if not log_files:
return
cutoff_date = datetime.now() - timedelta(days=keep_days)
deleted_count = 0
for log_file in log_files:
try:
# 从文件名提取日期(例如: unified_2024-01-15.log -> 2024-01-15)
filename = os.path.basename(log_file)
# 移除基础名称和下划线,移除.log扩展名
date_str = filename.replace(f"{base_name}_", "").replace(".log", "")
# 尝试解析日期
try:
file_date = datetime.strptime(date_str, "%Y-%m-%d")
except ValueError:
# 如果日期格式不匹配,尝试其他格式或跳过
continue
# 如果文件日期早于截止日期,删除文件
if file_date < cutoff_date:
os.remove(log_file)
deleted_count += 1
except Exception as e:
# 删除失败时记录但不中断流程
try:
print(f"[日志清理] 删除日志文件失败 {log_file}: {e}")
except Exception:
pass
if deleted_count > 0:
try:
print(
f"[日志清理] 已删除 {deleted_count} 个超过 {keep_days} 天的旧日志文件"
)
except Exception:
pass
except Exception as e:
# 清理失败不应影响程序运行
try:
print(f"[日志清理] 清理旧日志时出错: {e}")
except Exception:
pass
def get_logger(name: Optional[str] = None) -> logging.Logger:
"""获取指定名称的 logger,如果未配置则自动配置"""
if not _logging_configured:
configure_logging()
# Ensure name is never None
if name is None:
# 自动获取调用者模块名
frame = None
try:
frame = inspect.currentframe()
if frame and frame.f_back:
caller_frame = frame.f_back
name = caller_frame.f_globals.get("__name__", "root")
else:
name = "root"
except Exception:
name = "root"
finally:
if frame is not None:
try:
del frame
except Exception:
pass
# Final fallback to ensure name is a string
logger_name: str = name if name is not None else "root"
if logger_name not in _logger_cache:
_logger_cache[logger_name] = logging.getLogger(logger_name)
return _logger_cache[logger_name]
def configure_logging(
log_level: Optional[str] = None,
log_file: Optional[str] = None,
max_bytes: int = 10 * 1024 * 1024, # 10MB (保留作为备用)
backup_count: int = 5, # 保留作为备用
enable_console: bool = True,
enable_file: bool = True,
keep_days: Optional[int] = None, # 保留日志天数
use_dated_logs: Optional[bool] = None, # 是否使用按日期分割的日志
) -> None:
"""配置日志系统
参数:
log_level: 日志级别 (DEBUG, INFO, WARNING, ERROR, CRITICAL),默认从 config.json 读取
log_file: 日志文件基础路径,默认使用 logs/unified.log(会被转换为 logs/unified)
max_bytes: 单个日志文件最大大小(字节),超过后轮转(仅在不使用日期日志时生效)
backup_count: 保留的备份文件数量(仅在不使用日期日志时生效)
enable_console: 是否输出到控制台
enable_file: 是否输出到文件
keep_days: 保留日志的天数,超过此天数的日志将被自动删除,默认30天
use_dated_logs: 是否使用按日期分割的日志,默认True
"""
global _logging_configured
# 从 config.json 读取配置
if (
log_level is None
or log_file is None
or keep_days is None
or use_dated_logs is None
):
try:
import json
config_path = path_config.get_config_path()
if os.path.exists(config_path):
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
if log_level is None:
log_level = config.get("log_level", "INFO")
if log_file is None:
# 优先读取 log_unified_path,其次读取 logging.file_path
log_file = config.get("log_unified_path") or config.get("logging", {}).get("file_path", "logs/unified.log")
log_config = config.get("logging", {})
if keep_days is None:
keep_days = log_config.get("keep_days", 30)
if use_dated_logs is None:
use_dated_logs = log_config.get("use_dated_logs", True)
except Exception:
if log_level is None:
log_level = "INFO"
if log_file is None:
log_file = "logs/unified.log"
if keep_days is None:
keep_days = 30
if use_dated_logs is None:
use_dated_logs = True
# 规范化日志级别
if log_level is None:
log_level = "INFO"
numeric_level = getattr(logging, log_level.upper(), logging.INFO)
# 配置根 logger
root_logger = logging.getLogger()
root_logger.setLevel(numeric_level)
# 清除已有的 handlers(避免重复配置)
root_logger.handlers.clear()
# 统一日志格式:时间戳(毫秒) - [级别] - [线程名] - [模块名:函数名:行号] - [traceId] - 消息
detailed_format = (
"%(asctime)s.%(msecs)03d - [%(levelname)s] - [%(threadName)s] - "
"[%(name)s:%(funcName)s:%(lineno)d] - [traceId:%(traceId)s] - %(message)s"
)
simple_format = "%(asctime)s.%(msecs)03d - [%(levelname)s] - [%(threadName)s] - %(message)s"
# 控制台输出
if enable_console:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(numeric_level)
console_formatter = logging.Formatter(
simple_format, datefmt="%Y-%m-%d %H:%M:%S"
)
console_handler.setFormatter(console_formatter)
root_logger.addHandler(console_handler)
# 文件输出(按日期分割或传统轮转)
if enable_file:
try:
# 确保日志目录存在
if log_file is None:
log_file = "logs/unified.log" # 默认值
if not os.path.isabs(log_file):
log_file = os.path.join(path_config.PROJECT_ROOT, log_file)
# 如果是使用日期日志,提取基础文件名(移除扩展名)
if use_dated_logs:
# 从完整路径提取基础名称,例如: logs/unified.log -> logs/unified
if log_file:
base_filename = log_file.replace(".log", "")
else:
base_filename = "logs/unified"
log_dir = os.path.dirname(base_filename)
# 确保目录存在
if log_dir:
os.makedirs(log_dir, exist_ok=True)
# 使用按日期分割的Handler
file_handler = DatedFileHandler(
base_filename,
date_format="%Y-%m-%d",
encoding="utf-8-sig",
mode="a",
errors="replace",
)
# 启动时清理旧日志
if keep_days is not None:
cleanup_old_logs(
log_dir or path_config.get_logs_dir(),
os.path.basename(base_filename),
keep_days,
)
else:
# 使用传统的RotatingFileHandler
if log_file is None:
log_file = "logs/unified.log" # 默认值
log_dir = os.path.dirname(log_file)
if log_dir:
os.makedirs(log_dir, exist_ok=True)
file_handler = logging.handlers.RotatingFileHandler(
log_file, # type: ignore[arg-type]
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8-sig",
errors="replace",
)
file_handler.setLevel(numeric_level)
# 使用自定义Formatter以支持traceId
try:
from unified_logger import UnifiedFormatter
file_formatter = UnifiedFormatter(
detailed_format, datefmt="%Y-%m-%d %H:%M:%S"
)
except ImportError:
# 如果unified_logger不可用,使用标准Formatter
file_formatter = logging.Formatter(
detailed_format, datefmt="%Y-%m-%d %H:%M:%S"
)
# 添加traceId占位符处理
class TraceIdFormatter(logging.Formatter):
def format(self, record):
if not hasattr(record, 'traceId'):
record.traceId = 'N/A'
return super().format(record)
file_formatter = TraceIdFormatter(detailed_format, datefmt="%Y-%m-%d %H:%M:%S")
file_formatter.default_msec_format = "%s.%03d" # 毫秒精度
file_handler.setFormatter(file_formatter)
root_logger.addHandler(file_handler)
except Exception as e:
# 文件 handler 创建失败,只输出到控制台
try:
print(f"[日志配置] 警告:无法创建日志文件 {log_file}: {e}")
except Exception:
pass
_logging_configured = True
def log_function_call(
logger: Optional[logging.Logger] = None,
log_args: bool = True,
log_result: bool = False,
log_time: bool = True,
min_duration: float = 0.0, # 只记录超过此耗时的调用
):
"""装饰器:记录函数调用信息(参数、返回值、耗时)
参数:
logger: 使用的 logger,默认自动获取
log_args: 是否记录函数参数
log_result: 是否记录返回值
log_time: 是否记录执行时间
min_duration: 最小耗时阈值(秒),低于此值不记录耗时
"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs):
if logger is None:
_logger = get_logger()
else:
_logger = logger
func_name = func.__name__
start_time = time.time()
# 记录函数调用
if log_args:
args_str = ", ".join(
[str(arg)[:100] for arg in args[:3]]
) # 只记录前3个参数,避免过长
kwargs_str = ", ".join(
[f"{k}={str(v)[:50]}" for k, v in list(kwargs.items())[:3]]
)
call_info = (
f"{func_name}({args_str}{', ' + kwargs_str if kwargs_str else ''})"
)
if len(args) > 3 or len(kwargs) > 3:
call_info += "..."
_logger.debug(f"调用函数: {call_info}")
# 执行函数
try:
result = func(*args, **kwargs)
elapsed = time.time() - start_time
# 记录执行结果
if log_time and elapsed >= min_duration:
_logger.debug(f"函数 {func_name} 执行完成,耗时: {elapsed:.3f}秒")
if log_result:
result_str = str(result)[:200] # 限制长度
_logger.debug(f"函数 {func_name} 返回值: {result_str}")
return result
except Exception as e:
elapsed = time.time() - start_time
_logger.exception(
f"函数 {func_name} 执行异常,耗时: {elapsed:.3f}秒", exc_info=True
)
raise
return wrapper
return decorator
class LogContext:
"""上下文管理器:记录代码块执行时间"""
def __init__(
self,
message: str,
logger: Optional[logging.Logger] = None,
level: str = "DEBUG",
log_on_exit: bool = True,
):
self.message = message
self.logger = logger or get_logger()
self.level = getattr(logging, level.upper(), logging.DEBUG)
self.log_on_exit = log_on_exit
self.start_time: Optional[float] = None
def __enter__(self):
self.start_time = time.time()
self.logger.log(self.level, f"{self.message} [开始]")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if not self.log_on_exit:
return False
if self.start_time is None:
elapsed = 0.0
else:
elapsed = time.time() - self.start_time
if exc_type is None:
self.logger.log(self.level, f"{self.message} [完成] 耗时: {elapsed:.3f}秒")
else:
self.logger.log(
logging.ERROR,
f"{self.message} [异常] 耗时: {elapsed:.3f}秒",
exc_info=(exc_type, exc_val, exc_tb),
)
return False # 不抑制异常
def get_caller_info(skip_frames: int = 0) -> Dict[str, Any]:
"""获取调用者信息(模块名、函数名、行号)
参数:
skip_frames: 跳过的栈帧数(用于在装饰器中获取真正的调用者)
返回:
包含 module, function, line, file 的字典
"""
frame = None
try:
frame = inspect.currentframe()
if frame:
# 跳过当前函数和装饰器
for _ in range(skip_frames + 2):
if frame.f_back:
frame = frame.f_back
else:
break
if frame:
return {
"module": os.path.basename(frame.f_code.co_filename),
"function": frame.f_code.co_name,
"line": frame.f_lineno,
"file": frame.f_code.co_filename,
}
except Exception:
pass
finally:
if frame is not None:
try:
del frame
except Exception:
pass
return {
"module": "unknown",
"function": "unknown",
"line": 0,
"file": "unknown",
}
def format_exception_info(exc_info=None) -> str:
"""格式化异常信息为字符串
参数:
exc_info: 异常信息元组 (type, value, traceback),默认使用 sys.exc_info()
返回:
格式化的异常堆栈字符串
"""
if exc_info is None:
exc_info = sys.exc_info()
if exc_info[0] is None:
return ""
try:
# exc_info 应该是 (type, value, traceback) 元组
if isinstance(exc_info, tuple) and len(exc_info) >= 3:
exc_type, exc_value, exc_tb = exc_info[0], exc_info[1], exc_info[2]
# 确保类型正确
if isinstance(exc_value, BaseException):
return "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
else:
# 如果不是异常实例,创建一个包装
return "".join(
traceback.format_exception(
exc_type,
(
BaseException(str(exc_value))
if exc_value
else BaseException("Unknown error")
),
exc_tb,
)
)
else:
return "".join(traceback.format_exception(*exc_info))
except Exception:
return (
f"异常: {exc_info[1] if len(exc_info) > 1 and exc_info[1] else 'Unknown'}"
)
# 自动配置日志(在模块导入时)
if not _logging_configured:
# 延迟配置,避免在导入时读取 config.json 失败
try:
configure_logging()
except Exception:
pass