- Removed the `.coverage` file from the repository to streamline coverage management. - Updated `.gitignore` to exclude coverage-related files and directories, ensuring a cleaner project structure. - Introduced new scripts for checking GPU availability and detecting GUI parameter conflicts, enhancing the overall functionality of the application. - Improved logging mechanisms across various modules to support better debugging and monitoring. - Added comprehensive documentation for new features and updated existing markdown files for clarity.
488 lines
17 KiB
Python
488 lines
17 KiB
Python
"""
|
||
统一日志系统模块
|
||
================
|
||
|
||
功能特性:
|
||
1. 统一的日志格式规范(时间戳、级别、线程、类名、行号、traceId)
|
||
2. MDC(Mapped Diagnostic Context)支持,用于追踪请求链路
|
||
3. 敏感信息自动脱敏(密码、身份证、手机号等)
|
||
4. 性能优化(DEBUG级别判断、避免不必要的字符串拼接)
|
||
5. 异常堆栈完整记录
|
||
6. 方法入口/出口自动记录
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
import re
|
||
import time
|
||
import uuid
|
||
import logging
|
||
import threading
|
||
import inspect
|
||
import traceback
|
||
from typing import Optional, Dict, Any, Callable
|
||
from functools import wraps
|
||
from contextlib import contextmanager
|
||
|
||
import douyin_logging
|
||
|
||
|
||
# 线程本地存储,用于MDC(Mapped Diagnostic Context)
|
||
_thread_local = threading.local()
|
||
|
||
|
||
class MDC:
|
||
"""Mapped Diagnostic Context - 用于在日志中传递上下文信息(如traceId)"""
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def put(key: str, value: Any) -> None:
|
||
"""设置MDC值"""
|
||
if not hasattr(_thread_local, 'mdc'):
|
||
_thread_local.mdc = {}
|
||
_thread_local.mdc[key] = value
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def get(key: str, default: Any = None) -> Any:
|
||
"""获取MDC值"""
|
||
if hasattr(_thread_local, 'mdc'):
|
||
return _thread_local.mdc.get(key, default)
|
||
return default
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def remove(key: str) -> None:
|
||
"""移除MDC值"""
|
||
if hasattr(_thread_local, 'mdc'):
|
||
_thread_local.mdc.pop(key, None)
|
||
|
||
@staticmethod
|
||
def clear() -> None:
|
||
"""清空所有MDC值"""
|
||
if hasattr(_thread_local, 'mdc'):
|
||
_thread_local.mdc.clear()
|
||
|
||
@staticmethod
|
||
def get_all() -> Dict[str, Any]:
|
||
"""获取所有MDC值"""
|
||
if hasattr(_thread_local, 'mdc'):
|
||
return dict(_thread_local.mdc)
|
||
return {}
|
||
|
||
@staticmethod
|
||
def get_trace_id() -> Optional[str]:
|
||
"""获取当前线程的traceId"""
|
||
result = MDC.get('traceId')
|
||
return result if result else None
|
||
|
||
@staticmethod
|
||
def set_trace_id(trace_id: Optional[str] = None) -> str:
|
||
"""设置当前线程的traceId,如果未提供则自动生成"""
|
||
if trace_id is None:
|
||
trace_id = str(uuid.uuid4())[:16] # 使用UUID前16位作为traceId
|
||
MDC.put('traceId', trace_id)
|
||
return str(trace_id) # 确保返回 str 类型
|
||
|
||
|
||
class SensitiveDataMasker:
|
||
"""敏感信息脱敏工具"""
|
||
|
||
# 敏感信息模式
|
||
PATTERNS = {
|
||
'password': [
|
||
r'(?i)(password|pwd|passwd)\s*[:=]\s*([^\s,;}\]]+)',
|
||
r'(?i)(password|pwd|passwd)["\']?\s*[:=]\s*["\']?([^"\'\s,;}\]]+)',
|
||
],
|
||
'phone': [
|
||
r'(\d{3})\d{4}(\d{4})', # 手机号:保留前3位和后4位
|
||
r'(\+86\s*)?(\d{3})\d{4}(\d{4})', # 带区号的手机号
|
||
],
|
||
'id_card': [
|
||
r'(\d{6})\d{8}(\d{4})', # 身份证号:保留前6位和后4位
|
||
r'(\d{3})\d{11}(\d{3})', # 18位身份证号
|
||
],
|
||
'email': [
|
||
r'([a-zA-Z0-9._%+-]+)@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', # 邮箱:保留@前1位和域名
|
||
],
|
||
'bank_card': [
|
||
r'(\d{4})\d{8,12}(\d{4})', # 银行卡号:保留前4位和后4位
|
||
],
|
||
}
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_password(text: str) -> str:
|
||
"""脱敏密码"""
|
||
for pattern in SensitiveDataMasker.PATTERNS['password']:
|
||
text = re.sub(pattern, r'\1=***', text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_phone(text: str) -> str:
|
||
"""脱敏手机号"""
|
||
# 手机号脱敏:保留前3位和后4位
|
||
text = re.sub(r'(\d{3})\d{4}(\d{4})', r'\1****\2', text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_id_card(text: str) -> str:
|
||
"""脱敏身份证号"""
|
||
# 18位身份证:保留前6位和后4位
|
||
text = re.sub(r'(\d{6})\d{8}(\d{4})', r'\1********\2', text)
|
||
# 15位身份证:保留前6位和后3位
|
||
text = re.sub(r'(\d{6})\d{6}(\d{3})', r'\1******\2', text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_email(text: str) -> str:
|
||
"""脱敏邮箱"""
|
||
def replace_email(match):
|
||
username = match.group(1)
|
||
domain = match.group(2)
|
||
if len(username) > 1:
|
||
masked_username = username[0] + '*' * (len(username) - 1)
|
||
else:
|
||
masked_username = '*'
|
||
return f'{masked_username}@{domain}'
|
||
|
||
text = re.sub(SensitiveDataMasker.PATTERNS['email'][0], replace_email, text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_bank_card(text: str) -> str:
|
||
"""脱敏银行卡号"""
|
||
text = re.sub(r'(\d{4})\d{8,12}(\d{4})', r'\1********\2', text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_all(text: str) -> str:
|
||
"""对所有敏感信息进行脱敏(性能优化版本)"""
|
||
if not isinstance(text, str):
|
||
text = str(text)
|
||
|
||
# 优化:快速检查,如果文本很短或明显不包含敏感信息,跳过部分脱敏
|
||
text_lower = text.lower()
|
||
has_sensitive_keywords = any(
|
||
keyword in text_lower
|
||
for keyword in ['password', 'pwd', 'phone', 'mobile', 'id', 'card', '@']
|
||
)
|
||
|
||
# 只在可能包含敏感信息时才执行脱敏
|
||
if has_sensitive_keywords:
|
||
text = SensitiveDataMasker.mask_password(text)
|
||
text = SensitiveDataMasker.mask_phone(text)
|
||
text = SensitiveDataMasker.mask_id_card(text)
|
||
text = SensitiveDataMasker.mask_email(text)
|
||
text = SensitiveDataMasker.mask_bank_card(text)
|
||
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_dict(data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""对字典中的敏感字段进行脱敏"""
|
||
sensitive_keys = ['password', 'pwd', 'passwd', 'phone', 'mobile', 'tel',
|
||
'id_card', 'idcard', 'email', 'bank_card', 'bankcard',
|
||
'card_no', 'card_number', 'secret', 'token', 'api_key']
|
||
|
||
masked_data = {}
|
||
for key, value in data.items():
|
||
if any(sk in key.lower() for sk in sensitive_keys):
|
||
if isinstance(value, str):
|
||
masked_data[key] = SensitiveDataMasker.mask_all(value)
|
||
else:
|
||
masked_data[key] = '***'
|
||
elif isinstance(value, dict):
|
||
masked_data[key] = SensitiveDataMasker.mask_dict(value)
|
||
elif isinstance(value, str):
|
||
masked_data[key] = SensitiveDataMasker.mask_all(value)
|
||
else:
|
||
masked_data[key] = value
|
||
|
||
return masked_data
|
||
|
||
|
||
class UnifiedFormatter(logging.Formatter):
|
||
"""统一的日志格式化器,包含traceId和MDC信息"""
|
||
|
||
def __init__(self, *args, **kwargs):
|
||
super().__init__(*args, **kwargs)
|
||
# 标准格式:时间戳(毫秒) - [级别] - [线程名] - [模块名:函数名:行号] - [traceId] - 消息
|
||
self._detailed_format = (
|
||
"%(asctime)s.%(msecs)03d - [%(levelname)s] - [%(threadName)s] - "
|
||
"[%(name)s:%(funcName)s:%(lineno)d] - [traceId:%(traceId)s] - %(message)s"
|
||
)
|
||
# 缓存:避免重复检查MDC
|
||
self._last_mdc_check_time = 0.0
|
||
self._last_mdc_data = {}
|
||
self._mdc_cache_ttl = 0.1 # 100ms缓存,减少MDC查询频率(在高频日志场景下提升性能)
|
||
self._mdc_check_counter = 0 # 计数器,用于进一步优化
|
||
|
||
def format(self, record: logging.LogRecord) -> str:
|
||
"""格式化日志记录,添加traceId(性能优化版本)"""
|
||
# 获取traceId(快速路径,直接访问thread-local,无需调用get_all)
|
||
trace_id = MDC.get_trace_id()
|
||
if trace_id is None:
|
||
trace_id = 'N/A'
|
||
|
||
# 将traceId添加到record中
|
||
record.traceId = trace_id # type: ignore[attr-defined]
|
||
|
||
# 优化:使用计数器和时间戳双重检查,减少MDC查询频率
|
||
self._mdc_check_counter += 1
|
||
current_time = time.time()
|
||
should_check_mdc = (
|
||
(current_time - self._last_mdc_check_time) > self._mdc_cache_ttl or
|
||
self._mdc_check_counter >= 50 # 每50条记录强制检查一次
|
||
)
|
||
|
||
if should_check_mdc:
|
||
self._last_mdc_data = MDC.get_all()
|
||
self._last_mdc_check_time = current_time
|
||
self._mdc_check_counter = 0
|
||
|
||
mdc_data = self._last_mdc_data
|
||
if mdc_data:
|
||
# 将MDC信息添加到消息中(排除traceId,避免重复)
|
||
mdc_items = [(k, v) for k, v in mdc_data.items() if k != 'traceId']
|
||
if mdc_items:
|
||
# 优化:使用列表推导式,避免多次字符串拼接
|
||
mdc_str = ', '.join(f'{k}={v}' for k, v in mdc_items)
|
||
if mdc_str:
|
||
record.msg = f"{record.msg} | MDC: {mdc_str}"
|
||
|
||
# 优化:只在消息较长或可能包含敏感信息时才进行脱敏
|
||
# 进一步优化:缓存敏感关键词检查结果(对于相同消息)
|
||
if isinstance(record.msg, str):
|
||
msg = record.msg
|
||
# 快速检查:如果消息很短(<30字符),直接跳过脱敏(提高阈值以减少检查)
|
||
if len(msg) > 30:
|
||
# 使用更高效的检查:先检查长度,再检查关键词
|
||
msg_lower = msg.lower()
|
||
if any(keyword in msg_lower for keyword in
|
||
['password', 'pwd', 'phone', 'mobile', 'id', 'card', 'email', '@']):
|
||
record.msg = SensitiveDataMasker.mask_all(msg)
|
||
|
||
# 格式化
|
||
return super().format(record)
|
||
|
||
|
||
# 缓存logger实例,避免重复创建和检查
|
||
_logger_cache: Dict[str, logging.Logger] = {}
|
||
_logger_cache_lock = threading.Lock()
|
||
|
||
def get_unified_logger(name: Optional[str] = None) -> logging.Logger:
|
||
"""获取统一配置的logger实例(性能优化版本)"""
|
||
# 确保name不为None
|
||
if name is None:
|
||
# 优化:避免使用inspect.currentframe(),直接使用调用栈
|
||
# 如果调用者没有提供name,使用默认值
|
||
name = "root"
|
||
try:
|
||
# 尝试从调用栈获取模块名(最多向上查找3层)
|
||
frame = inspect.currentframe()
|
||
if frame:
|
||
for _ in range(3): # 最多向上查找3层
|
||
frame = frame.f_back
|
||
if frame is None:
|
||
break
|
||
module_name = frame.f_globals.get("__name__")
|
||
if module_name and module_name != __name__:
|
||
name = module_name
|
||
break
|
||
except Exception:
|
||
# 如果获取失败,使用默认值
|
||
pass
|
||
|
||
# 确保name是str类型(类型检查)
|
||
logger_name: str = name if isinstance(name, str) else "root"
|
||
|
||
# 使用缓存避免重复创建和检查
|
||
with _logger_cache_lock:
|
||
if logger_name in _logger_cache:
|
||
return _logger_cache[logger_name]
|
||
|
||
logger = douyin_logging.get_logger(logger_name)
|
||
|
||
# 如果logger还没有自定义formatter,添加统一formatter
|
||
# 优化:只检查一次,避免每次调用都检查
|
||
if logger.handlers:
|
||
for handler in logger.handlers:
|
||
if isinstance(handler.formatter, logging.Formatter) and \
|
||
not isinstance(handler.formatter, UnifiedFormatter):
|
||
handler.setFormatter(UnifiedFormatter(
|
||
fmt=handler.formatter._fmt if hasattr(handler.formatter, '_fmt') else None,
|
||
datefmt="%Y-%m-%d %H:%M:%S"
|
||
))
|
||
|
||
_logger_cache[logger_name] = logger
|
||
return logger
|
||
|
||
|
||
@contextmanager
|
||
def trace_context(trace_id: Optional[str] = None, **mdc_context):
|
||
"""上下文管理器:为代码块设置traceId和MDC上下文"""
|
||
old_trace_id = MDC.get_trace_id()
|
||
old_mdc = MDC.get_all()
|
||
|
||
# 设置新的traceId
|
||
if trace_id is None:
|
||
trace_id = MDC.set_trace_id()
|
||
else:
|
||
MDC.set_trace_id(trace_id)
|
||
|
||
# 设置其他MDC上下文
|
||
for key, value in mdc_context.items():
|
||
MDC.put(key, value)
|
||
|
||
try:
|
||
yield trace_id
|
||
finally:
|
||
# 恢复旧的MDC
|
||
MDC.clear()
|
||
if old_trace_id:
|
||
MDC.set_trace_id(old_trace_id)
|
||
for key, value in old_mdc.items():
|
||
MDC.put(key, value)
|
||
|
||
|
||
def log_method_entry_exit(logger: Optional[logging.Logger] = None,
|
||
log_args: bool = True,
|
||
log_result: bool = False,
|
||
log_duration: bool = True,
|
||
min_duration: float = 0.0):
|
||
"""装饰器:自动记录方法入口、出口、参数、返回值、耗时"""
|
||
def decorator(func: Callable) -> Callable:
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
if logger is None:
|
||
_logger = get_unified_logger()
|
||
else:
|
||
_logger = logger
|
||
|
||
func_name = f"{func.__module__}.{func.__name__}"
|
||
start_time = time.time()
|
||
|
||
# 记录方法入口
|
||
if log_args:
|
||
args_str = ", ".join([str(arg)[:100] for arg in args[:3]])
|
||
kwargs_str = ", ".join([f"{k}={str(v)[:50]}" for k, v in list(kwargs.items())[:3]])
|
||
call_info = f"{func_name}({args_str}{', ' + kwargs_str if kwargs_str else ''})"
|
||
if len(args) > 3 or len(kwargs) > 3:
|
||
call_info += "..."
|
||
_logger.info(f"[方法入口] {call_info}")
|
||
|
||
# 执行方法
|
||
try:
|
||
result = func(*args, **kwargs)
|
||
elapsed = time.time() - start_time
|
||
|
||
# 记录方法出口
|
||
exit_msg = f"[方法出口] {func_name}"
|
||
if log_duration and elapsed >= min_duration:
|
||
exit_msg += f" 耗时: {elapsed:.3f}秒"
|
||
if log_result:
|
||
result_str = str(result)[:200]
|
||
exit_msg += f" 返回值: {result_str}"
|
||
|
||
_logger.info(exit_msg)
|
||
|
||
return result
|
||
except Exception as e:
|
||
elapsed = time.time() - start_time
|
||
_logger.error(
|
||
f"[方法异常] {func_name} 发生异常: {type(e).__name__}: {str(e)} "
|
||
f"耗时: {elapsed:.3f}秒",
|
||
exc_info=True
|
||
)
|
||
raise
|
||
|
||
return wrapper
|
||
return decorator
|
||
|
||
|
||
def log_external_call(target: str,
|
||
request_params: Optional[Dict[str, Any]] = None,
|
||
response_result: Optional[Any] = None,
|
||
success: bool = True,
|
||
duration: Optional[float] = None,
|
||
logger: Optional[logging.Logger] = None):
|
||
"""记录外部调用(API调用、数据库查询等)"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
# 脱敏请求参数
|
||
if request_params:
|
||
request_params = SensitiveDataMasker.mask_dict(request_params)
|
||
|
||
msg_parts = [f"[外部调用] {target}"]
|
||
|
||
if request_params:
|
||
params_str = ", ".join([f"{k}={v}" for k, v in list(request_params.items())[:5]])
|
||
if len(request_params) > 5:
|
||
params_str += "..."
|
||
msg_parts.append(f"请求参数: {params_str}")
|
||
|
||
if response_result is not None:
|
||
result_str = str(response_result)[:200]
|
||
msg_parts.append(f"响应结果: {result_str}")
|
||
|
||
if duration is not None:
|
||
msg_parts.append(f"耗时: {duration:.3f}秒")
|
||
|
||
msg_parts.append(f"状态: {'成功' if success else '失败'}")
|
||
|
||
level = logging.INFO if success else logging.ERROR
|
||
logger.log(level, " | ".join(msg_parts))
|
||
|
||
|
||
def log_decision_point(decision: str,
|
||
condition: Optional[str] = None,
|
||
result: Any = None,
|
||
logger: Optional[logging.Logger] = None):
|
||
"""记录关键决策点(分支判断、条件判断等)"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
msg_parts = [f"[决策点] {decision}"]
|
||
|
||
if condition:
|
||
msg_parts.append(f"条件: {condition}")
|
||
|
||
if result is not None:
|
||
result_str = str(result)[:100]
|
||
msg_parts.append(f"结果: {result_str}")
|
||
|
||
logger.info(" | ".join(msg_parts))
|
||
|
||
|
||
def log_state_change(old_state: Any,
|
||
new_state: Any,
|
||
reason: Optional[str] = None,
|
||
logger: Optional[logging.Logger] = None):
|
||
"""记录状态变化"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
msg_parts = [
|
||
f"[状态变化] {old_state} -> {new_state}"
|
||
]
|
||
|
||
if reason:
|
||
msg_parts.append(f"原因: {reason}")
|
||
|
||
logger.info(" | ".join(msg_parts))
|
||
|
||
|
||
# 导出主要接口
|
||
__all__ = [
|
||
'MDC',
|
||
'SensitiveDataMasker',
|
||
'UnifiedFormatter',
|
||
'get_unified_logger',
|
||
'trace_context',
|
||
'log_method_entry_exit',
|
||
'log_external_call',
|
||
'log_decision_point',
|
||
'log_state_change',
|
||
]
|
||
|