- Removed the reservation preempt minimum value from the Runner and GUI components. - Updated logging mechanism to streamline log message handling and improve readability. - Introduced a new method for appending log lines in the Runner class. - Cleaned up related GUI elements and configuration options in web interface. - Deleted obsolete simulation scripts related to reservation preemption checks.
540 lines
18 KiB
Python
540 lines
18 KiB
Python
"""
|
||
统一日志系统模块
|
||
================
|
||
|
||
功能特性:
|
||
1. 统一的日志格式规范(时间戳、级别、线程、类名、行号、traceId)
|
||
2. MDC(Mapped Diagnostic Context)支持,用于追踪请求链路
|
||
3. 敏感信息自动脱敏(密码、身份证、手机号等)
|
||
4. 性能优化(DEBUG级别判断、避免不必要的字符串拼接)
|
||
5. 异常堆栈完整记录
|
||
6. 方法入口/出口自动记录
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
import re
|
||
import time
|
||
import uuid
|
||
import logging
|
||
import threading
|
||
import inspect
|
||
import traceback
|
||
from typing import Optional, Dict, Any, Callable
|
||
from functools import wraps
|
||
from contextlib import contextmanager
|
||
|
||
from douyin.logging import douyin_logging
|
||
|
||
|
||
# 线程本地存储,用于MDC(Mapped Diagnostic Context)
|
||
_thread_local = threading.local()
|
||
|
||
|
||
class MDC:
|
||
"""Mapped Diagnostic Context - 用于在日志中传递上下文信息(如traceId)"""
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def put(key: str, value: Any) -> None:
|
||
"""设置MDC值"""
|
||
if not hasattr(_thread_local, "mdc"):
|
||
_thread_local.mdc = {}
|
||
_thread_local.mdc[key] = value
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def get(key: str, default: Any = None) -> Any:
|
||
"""获取MDC值"""
|
||
if hasattr(_thread_local, "mdc"):
|
||
return _thread_local.mdc.get(key, default)
|
||
return default
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def remove(key: str) -> None:
|
||
"""移除MDC值"""
|
||
if hasattr(_thread_local, "mdc"):
|
||
_thread_local.mdc.pop(key, None)
|
||
|
||
@staticmethod
|
||
def clear() -> None:
|
||
"""清空所有MDC值"""
|
||
if hasattr(_thread_local, "mdc"):
|
||
_thread_local.mdc.clear()
|
||
|
||
@staticmethod
|
||
def get_all() -> Dict[str, Any]:
|
||
"""获取所有MDC值"""
|
||
if hasattr(_thread_local, "mdc"):
|
||
return dict(_thread_local.mdc)
|
||
return {}
|
||
|
||
@staticmethod
|
||
def get_trace_id() -> Optional[str]:
|
||
"""获取当前线程的traceId"""
|
||
result = MDC.get("traceId")
|
||
return result if result else None
|
||
|
||
@staticmethod
|
||
def set_trace_id(trace_id: Optional[str] = None) -> str:
|
||
"""设置当前线程的traceId,如果未提供则自动生成"""
|
||
if trace_id is None:
|
||
trace_id = str(uuid.uuid4())[:16] # 使用UUID前16位作为traceId
|
||
MDC.put("traceId", trace_id)
|
||
return str(trace_id) # 确保返回 str 类型
|
||
|
||
|
||
class SensitiveDataMasker:
|
||
"""敏感信息脱敏工具"""
|
||
|
||
# 敏感信息模式
|
||
PATTERNS = {
|
||
"password": [
|
||
r"(?i)(password|pwd|passwd)\s*[:=]\s*([^\s,;}\]]+)",
|
||
r'(?i)(password|pwd|passwd)["\']?\s*[:=]\s*["\']?([^"\'\s,;}\]]+)',
|
||
],
|
||
"phone": [
|
||
r"(\d{3})\d{4}(\d{4})", # 手机号:保留前3位和后4位
|
||
r"(\+86\s*)?(\d{3})\d{4}(\d{4})", # 带区号的手机号
|
||
],
|
||
"id_card": [
|
||
r"(\d{6})\d{8}(\d{4})", # 身份证号:保留前6位和后4位
|
||
r"(\d{3})\d{11}(\d{3})", # 18位身份证号
|
||
],
|
||
"email": [
|
||
r"([a-zA-Z0-9._%+-]+)@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})", # 邮箱:保留@前1位和域名
|
||
],
|
||
"bank_card": [
|
||
r"(\d{4})\d{8,12}(\d{4})", # 银行卡号:保留前4位和后4位
|
||
],
|
||
}
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_password(text: str) -> str:
|
||
"""脱敏密码"""
|
||
for pattern in SensitiveDataMasker.PATTERNS["password"]:
|
||
text = re.sub(pattern, r"\1=***", text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_phone(text: str) -> str:
|
||
"""脱敏手机号"""
|
||
# 手机号脱敏:保留前3位和后4位
|
||
text = re.sub(r"(\d{3})\d{4}(\d{4})", r"\1****\2", text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_id_card(text: str) -> str:
|
||
"""脱敏身份证号"""
|
||
# 18位身份证:保留前6位和后4位
|
||
text = re.sub(r"(\d{6})\d{8}(\d{4})", r"\1********\2", text)
|
||
# 15位身份证:保留前6位和后3位
|
||
text = re.sub(r"(\d{6})\d{6}(\d{3})", r"\1******\2", text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_email(text: str) -> str:
|
||
"""脱敏邮箱"""
|
||
|
||
def replace_email(match):
|
||
username = match.group(1)
|
||
domain = match.group(2)
|
||
if len(username) > 1:
|
||
masked_username = username[0] + "*" * (len(username) - 1)
|
||
else:
|
||
masked_username = "*"
|
||
return f"{masked_username}@{domain}"
|
||
|
||
text = re.sub(SensitiveDataMasker.PATTERNS["email"][0], replace_email, text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_bank_card(text: str) -> str:
|
||
"""脱敏银行卡号"""
|
||
text = re.sub(r"(\d{4})\d{8,12}(\d{4})", r"\1********\2", text)
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_all(text: str) -> str:
|
||
"""对所有敏感信息进行脱敏(性能优化版本)"""
|
||
if not isinstance(text, str):
|
||
text = str(text)
|
||
|
||
# 优化:快速检查,如果文本很短或明显不包含敏感信息,跳过部分脱敏
|
||
text_lower = text.lower()
|
||
has_sensitive_keywords = any(
|
||
keyword in text_lower
|
||
for keyword in ["password", "pwd", "phone", "mobile", "id", "card", "@"]
|
||
)
|
||
|
||
# 只在可能包含敏感信息时才执行脱敏
|
||
if has_sensitive_keywords:
|
||
text = SensitiveDataMasker.mask_password(text)
|
||
text = SensitiveDataMasker.mask_phone(text)
|
||
text = SensitiveDataMasker.mask_id_card(text)
|
||
text = SensitiveDataMasker.mask_email(text)
|
||
text = SensitiveDataMasker.mask_bank_card(text)
|
||
|
||
return text
|
||
|
||
@staticmethod # type: ignore[misc]
|
||
def mask_dict(data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""对字典中的敏感字段进行脱敏"""
|
||
sensitive_keys = [
|
||
"password",
|
||
"pwd",
|
||
"passwd",
|
||
"phone",
|
||
"mobile",
|
||
"tel",
|
||
"id_card",
|
||
"idcard",
|
||
"email",
|
||
"bank_card",
|
||
"bankcard",
|
||
"card_no",
|
||
"card_number",
|
||
"secret",
|
||
"token",
|
||
"api_key",
|
||
]
|
||
|
||
masked_data = {}
|
||
for key, value in data.items():
|
||
if any(sk in key.lower() for sk in sensitive_keys):
|
||
if isinstance(value, str):
|
||
masked_data[key] = SensitiveDataMasker.mask_all(value)
|
||
else:
|
||
masked_data[key] = "***"
|
||
elif isinstance(value, dict):
|
||
masked_data[key] = SensitiveDataMasker.mask_dict(value)
|
||
elif isinstance(value, str):
|
||
masked_data[key] = SensitiveDataMasker.mask_all(value)
|
||
else:
|
||
masked_data[key] = value
|
||
|
||
return masked_data
|
||
|
||
|
||
class UnifiedFormatter(logging.Formatter):
|
||
"""统一的日志格式化器,包含traceId和MDC信息"""
|
||
|
||
def __init__(self, *args, **kwargs):
|
||
super().__init__(*args, **kwargs)
|
||
# 简化的格式:时间戳 - [级别] - [模块] - 消息
|
||
# 去掉了冗余的线程名、函数名、行号、traceId等信息,使日志更简洁
|
||
self._detailed_format = (
|
||
"%(asctime)s - [%(levelname)s] - [%(name)s] - %(message)s"
|
||
)
|
||
# 缓存:避免重复检查MDC
|
||
self._last_mdc_check_time = 0.0
|
||
self._last_mdc_data = {}
|
||
self._mdc_cache_ttl = (
|
||
0.1 # 100ms缓存,减少MDC查询频率(在高频日志场景下提升性能)
|
||
)
|
||
self._mdc_check_counter = 0 # 计数器,用于进一步优化
|
||
|
||
def format(self, record: logging.LogRecord) -> str:
|
||
"""格式化日志记录,简化显示以提高可读性"""
|
||
# 只在DEBUG级别或错误时显示traceId
|
||
if record.levelno <= logging.DEBUG or record.levelno >= logging.ERROR:
|
||
trace_id = MDC.get_trace_id()
|
||
if trace_id:
|
||
record.traceId = trace_id # type: ignore[attr-defined]
|
||
else:
|
||
record.traceId = "N/A" # type: ignore[attr-defined]
|
||
else:
|
||
record.traceId = "" # type: ignore[attr-defined]
|
||
|
||
# 只在DEBUG级别显示完整的MDC信息
|
||
if record.levelno <= logging.DEBUG:
|
||
# 优化:使用计数器和时间戳双重检查,减少MDC查询频率
|
||
self._mdc_check_counter += 1
|
||
current_time = time.time()
|
||
should_check_mdc = (
|
||
current_time - self._last_mdc_check_time
|
||
) > self._mdc_cache_ttl or self._mdc_check_counter >= 50 # 每50条记录强制检查一次
|
||
|
||
if should_check_mdc:
|
||
self._last_mdc_data = MDC.get_all()
|
||
self._last_mdc_check_time = current_time
|
||
self._mdc_check_counter = 0
|
||
|
||
mdc_data = self._last_mdc_data
|
||
if mdc_data:
|
||
# 将MDC信息添加到消息中(排除traceId,避免重复)
|
||
mdc_items = [(k, v) for k, v in mdc_data.items() if k != "traceId"]
|
||
if mdc_items:
|
||
# 优化:使用列表推导式,避免多次字符串拼接
|
||
mdc_str = ", ".join(f"{k}={v}" for k, v in mdc_items)
|
||
if mdc_str:
|
||
record.msg = f"{record.msg} | MDC: {mdc_str}"
|
||
|
||
# 敏感信息脱敏(只对可能包含敏感信息的消息进行)
|
||
if isinstance(record.msg, str):
|
||
msg = record.msg
|
||
# 快速检查:如果消息很短(<50字符),直接跳过脱敏
|
||
if len(msg) > 50:
|
||
# 使用更高效的检查:先检查长度,再检查关键词
|
||
msg_lower = msg.lower()
|
||
if any(
|
||
keyword in msg_lower
|
||
for keyword in [
|
||
"password",
|
||
"pwd",
|
||
"phone",
|
||
"mobile",
|
||
"id",
|
||
"card",
|
||
"email",
|
||
"@",
|
||
]
|
||
):
|
||
record.msg = SensitiveDataMasker.mask_all(msg)
|
||
|
||
# 格式化
|
||
return super().format(record)
|
||
|
||
|
||
# 缓存logger实例,避免重复创建和检查
|
||
_logger_cache: Dict[str, logging.Logger] = {}
|
||
_logger_cache_lock = threading.Lock()
|
||
|
||
|
||
def get_unified_logger(name: Optional[str] = None) -> logging.Logger:
|
||
"""获取统一配置的logger实例(性能优化版本)"""
|
||
# 确保name不为None
|
||
if name is None:
|
||
# 优化:避免使用inspect.currentframe(),直接使用调用栈
|
||
# 如果调用者没有提供name,使用默认值
|
||
name = "root"
|
||
try:
|
||
# 尝试从调用栈获取模块名(最多向上查找3层)
|
||
frame = inspect.currentframe()
|
||
if frame:
|
||
for _ in range(3): # 最多向上查找3层
|
||
frame = frame.f_back
|
||
if frame is None:
|
||
break
|
||
module_name = frame.f_globals.get("__name__")
|
||
if module_name and module_name != __name__:
|
||
name = module_name
|
||
break
|
||
except Exception:
|
||
# 如果获取失败,使用默认值
|
||
pass
|
||
|
||
# 确保name是str类型(类型检查)
|
||
logger_name: str = name if isinstance(name, str) else "root"
|
||
|
||
# 使用缓存避免重复创建和检查
|
||
with _logger_cache_lock:
|
||
if logger_name in _logger_cache:
|
||
return _logger_cache[logger_name]
|
||
|
||
logger = douyin_logging.get_logger(logger_name)
|
||
|
||
# 如果logger还没有自定义formatter,添加统一formatter
|
||
# 优化:只检查一次,避免每次调用都检查
|
||
if logger.handlers:
|
||
for handler in logger.handlers:
|
||
if isinstance(handler.formatter, logging.Formatter) and not isinstance(
|
||
handler.formatter, UnifiedFormatter
|
||
):
|
||
handler.setFormatter(
|
||
UnifiedFormatter(
|
||
fmt=(
|
||
handler.formatter._fmt
|
||
if hasattr(handler.formatter, "_fmt")
|
||
else None
|
||
),
|
||
datefmt="%Y-%m-%d %H:%M:%S",
|
||
)
|
||
)
|
||
|
||
_logger_cache[logger_name] = logger
|
||
return logger
|
||
|
||
|
||
@contextmanager
|
||
def trace_context(trace_id: Optional[str] = None, **mdc_context):
|
||
"""上下文管理器:为代码块设置traceId和MDC上下文"""
|
||
old_trace_id = MDC.get_trace_id()
|
||
old_mdc = MDC.get_all()
|
||
|
||
# 设置新的traceId
|
||
if trace_id is None:
|
||
trace_id = MDC.set_trace_id()
|
||
else:
|
||
MDC.set_trace_id(trace_id)
|
||
|
||
# 设置其他MDC上下文
|
||
for key, value in mdc_context.items():
|
||
MDC.put(key, value)
|
||
|
||
try:
|
||
yield trace_id
|
||
finally:
|
||
# 恢复旧的MDC
|
||
MDC.clear()
|
||
if old_trace_id:
|
||
MDC.set_trace_id(old_trace_id)
|
||
for key, value in old_mdc.items():
|
||
MDC.put(key, value)
|
||
|
||
|
||
def log_method_entry_exit(
|
||
logger: Optional[logging.Logger] = None,
|
||
log_args: bool = True,
|
||
log_result: bool = False,
|
||
log_duration: bool = True,
|
||
min_duration: float = 0.0,
|
||
):
|
||
"""装饰器:自动记录方法入口、出口、参数、返回值、耗时"""
|
||
|
||
def decorator(func: Callable) -> Callable:
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
if logger is None:
|
||
_logger = get_unified_logger()
|
||
else:
|
||
_logger = logger
|
||
|
||
func_name = f"{func.__module__}.{func.__name__}"
|
||
start_time = time.time()
|
||
|
||
# 记录方法入口
|
||
if log_args:
|
||
args_str = ", ".join([str(arg)[:100] for arg in args[:3]])
|
||
kwargs_str = ", ".join(
|
||
[f"{k}={str(v)[:50]}" for k, v in list(kwargs.items())[:3]]
|
||
)
|
||
call_info = (
|
||
f"{func_name}({args_str}{', ' + kwargs_str if kwargs_str else ''})"
|
||
)
|
||
if len(args) > 3 or len(kwargs) > 3:
|
||
call_info += "..."
|
||
_logger.info(f"[方法入口] {call_info}")
|
||
|
||
# 执行方法
|
||
try:
|
||
result = func(*args, **kwargs)
|
||
elapsed = time.time() - start_time
|
||
|
||
# 记录方法出口
|
||
exit_msg = f"[方法出口] {func_name}"
|
||
if log_duration and elapsed >= min_duration:
|
||
exit_msg += f" 耗时: {elapsed:.3f}秒"
|
||
if log_result:
|
||
result_str = str(result)[:200]
|
||
exit_msg += f" 返回值: {result_str}"
|
||
|
||
_logger.info(exit_msg)
|
||
|
||
return result
|
||
except Exception as e:
|
||
elapsed = time.time() - start_time
|
||
_logger.error(
|
||
f"[方法异常] {func_name} 发生异常: {type(e).__name__}: {str(e)} "
|
||
f"耗时: {elapsed:.3f}秒",
|
||
exc_info=True,
|
||
)
|
||
raise
|
||
|
||
return wrapper
|
||
|
||
return decorator
|
||
|
||
|
||
def log_external_call(
|
||
target: str,
|
||
request_params: Optional[Dict[str, Any]] = None,
|
||
response_result: Optional[Any] = None,
|
||
success: bool = True,
|
||
duration: Optional[float] = None,
|
||
logger: Optional[logging.Logger] = None,
|
||
):
|
||
"""记录外部调用(API调用、数据库查询等)"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
# 脱敏请求参数
|
||
if request_params:
|
||
request_params = SensitiveDataMasker.mask_dict(request_params)
|
||
|
||
msg_parts = [f"[外部调用] {target}"]
|
||
|
||
if request_params:
|
||
params_str = ", ".join(
|
||
[f"{k}={v}" for k, v in list(request_params.items())[:5]]
|
||
)
|
||
if len(request_params) > 5:
|
||
params_str += "..."
|
||
msg_parts.append(f"请求参数: {params_str}")
|
||
|
||
if response_result is not None:
|
||
result_str = str(response_result)[:200]
|
||
msg_parts.append(f"响应结果: {result_str}")
|
||
|
||
if duration is not None:
|
||
msg_parts.append(f"耗时: {duration:.3f}秒")
|
||
|
||
msg_parts.append(f"状态: {'成功' if success else '失败'}")
|
||
|
||
level = logging.INFO if success else logging.ERROR
|
||
logger.log(level, " | ".join(msg_parts))
|
||
|
||
|
||
def log_decision_point(
|
||
decision: str,
|
||
condition: Optional[str] = None,
|
||
result: Any = None,
|
||
logger: Optional[logging.Logger] = None,
|
||
):
|
||
"""记录关键决策点(分支判断、条件判断等)"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
msg_parts = [f"[决策点] {decision}"]
|
||
|
||
if condition:
|
||
msg_parts.append(f"条件: {condition}")
|
||
|
||
if result is not None:
|
||
result_str = str(result)[:100]
|
||
msg_parts.append(f"结果: {result_str}")
|
||
|
||
logger.info(" | ".join(msg_parts))
|
||
|
||
|
||
def log_state_change(
|
||
old_state: Any,
|
||
new_state: Any,
|
||
reason: Optional[str] = None,
|
||
logger: Optional[logging.Logger] = None,
|
||
):
|
||
"""记录状态变化"""
|
||
if logger is None:
|
||
logger = get_unified_logger()
|
||
|
||
msg_parts = [f"[状态变化] {old_state} -> {new_state}"]
|
||
|
||
if reason:
|
||
msg_parts.append(f"原因: {reason}")
|
||
|
||
logger.info(" | ".join(msg_parts))
|
||
|
||
|
||
# 导出主要接口
|
||
__all__ = [
|
||
"MDC",
|
||
"SensitiveDataMasker",
|
||
"UnifiedFormatter",
|
||
"get_unified_logger",
|
||
"trace_context",
|
||
"log_method_entry_exit",
|
||
"log_external_call",
|
||
"log_decision_point",
|
||
"log_state_change",
|
||
]
|