334 lines
12 KiB
Python
334 lines
12 KiB
Python
"""douyin_logging_pkg.monitor - 日志轮转 / 大小监控。
|
||
|
||
- LogRotationMonitor 轮转监控线程(按间隔检查日志大小并触发清理)
|
||
- start_log_monitor 启动监控(懒创建实例)
|
||
- stop_log_monitor 停止监控
|
||
- get_log_stats 汇总日志统计信息
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import glob
|
||
import logging
|
||
import os
|
||
import threading
|
||
import time
|
||
from datetime import datetime
|
||
from typing import Any, Callable, Dict, Optional
|
||
|
||
from douyin.core import path_config
|
||
|
||
from . import _common as _state
|
||
from .cleanup import (
|
||
_load_logging_cleanup_policy_from_config,
|
||
cleanup_old_logs,
|
||
)
|
||
|
||
|
||
class LogRotationMonitor:
|
||
"""日志轮转监控器:监控日志大小、轮转频率并发出告警"""
|
||
|
||
def __init__(
|
||
self,
|
||
logs_dir: str,
|
||
base_name: str,
|
||
warning_size_mb: float = 500.0, # 警告大小(MB)
|
||
critical_size_mb: float = 2000.0, # 严重大小(MB)
|
||
check_interval: int = 300, # 检查间隔(秒)
|
||
rotation_threshold: int = 10, # 轮转频率阈值(每天)
|
||
alert_callback: Optional[Callable[[str, str], None]] = None,
|
||
enable_runtime_cleanup: bool = False,
|
||
):
|
||
"""
|
||
参数:
|
||
logs_dir: 日志目录
|
||
base_name: 日志文件基础名称
|
||
warning_size_mb: 警告大小阈值(MB)
|
||
critical_size_mb: 严重大小阈值(MB)
|
||
check_interval: 检查间隔(秒)
|
||
rotation_threshold: 轮转频率阈值(每天)
|
||
alert_callback: 告警回调函数,参数:(level, message)
|
||
"""
|
||
self.logs_dir = logs_dir
|
||
self.base_name = base_name
|
||
self.warning_size_mb = warning_size_mb
|
||
self.critical_size_mb = critical_size_mb
|
||
self.check_interval = check_interval
|
||
self.rotation_threshold = rotation_threshold
|
||
self.alert_callback = alert_callback
|
||
|
||
self._running = False
|
||
self._thread: Optional[threading.Thread] = None
|
||
self._last_rotation_count = 0
|
||
self._last_check_time = datetime.now()
|
||
# 清理状态,避免并发触发大量清理操作
|
||
self._cleanup_lock = threading.Lock()
|
||
self._cleanup_running = False
|
||
# 是否允许在运行过程中触发自动清理(默认不允许)
|
||
self.enable_runtime_cleanup = bool(enable_runtime_cleanup)
|
||
|
||
# 告警输出限流:避免在运行时开启监控时刷屏
|
||
self._last_warning_print_ts = 0.0
|
||
self._last_critical_print_ts = 0.0
|
||
self._min_alert_print_interval_sec = 3600.0
|
||
|
||
def start(self) -> None:
|
||
"""启动监控器"""
|
||
if self._running:
|
||
return
|
||
|
||
self._running = True
|
||
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
||
self._thread.start()
|
||
|
||
def stop(self) -> None:
|
||
"""停止监控器"""
|
||
self._running = False
|
||
if self._thread and self._thread.is_alive():
|
||
self._thread.join(timeout=5)
|
||
|
||
def _monitor_loop(self) -> None:
|
||
"""监控循环"""
|
||
while self._running:
|
||
try:
|
||
self._check_logs()
|
||
time.sleep(self.check_interval)
|
||
except Exception as e:
|
||
if self.alert_callback:
|
||
self.alert_callback("ERROR", f"日志监控器异常: {e}")
|
||
|
||
def _check_logs(self) -> None:
|
||
"""检查日志状态"""
|
||
try:
|
||
# 收集所有日志文件
|
||
pattern = os.path.join(self.logs_dir, f"{self.base_name}_*.log*")
|
||
log_files = glob.glob(pattern)
|
||
|
||
if not log_files:
|
||
return
|
||
|
||
# 计算总大小
|
||
total_size = sum(os.path.getsize(f) for f in log_files if os.path.exists(f))
|
||
total_size_mb = total_size / 1024 / 1024
|
||
|
||
# 检查大小告警
|
||
if total_size_mb >= self.critical_size_mb:
|
||
message = f"日志总大小严重超限: {total_size_mb:.1f}MB >= {self.critical_size_mb}MB"
|
||
if self.alert_callback:
|
||
self.alert_callback("CRITICAL", message)
|
||
else:
|
||
now_ts = time.time()
|
||
if (
|
||
now_ts - self._last_critical_print_ts
|
||
>= self._min_alert_print_interval_sec
|
||
):
|
||
print(f"[日志监控] CRITICAL: {message}")
|
||
self._last_critical_print_ts = now_ts
|
||
|
||
elif total_size_mb >= self.warning_size_mb:
|
||
message = (
|
||
f"日志总大小警告: {total_size_mb:.1f}MB >= {self.warning_size_mb}MB"
|
||
)
|
||
if self.alert_callback:
|
||
self.alert_callback("WARNING", message)
|
||
else:
|
||
now_ts = time.time()
|
||
if (
|
||
now_ts - self._last_warning_print_ts
|
||
>= self._min_alert_print_interval_sec
|
||
):
|
||
print(f"[日志监控] WARNING: {message}")
|
||
self._last_warning_print_ts = now_ts
|
||
# 运行时不再自动触发清理,清理仅在启动阶段执行以避免中途 I/O 干扰
|
||
# 检查轮转频率
|
||
current_time = datetime.now()
|
||
if (
|
||
current_time - self._last_check_time
|
||
).total_seconds() >= 86400: # 24小时
|
||
# 计算今天的文件数量
|
||
today_str = current_time.strftime("%Y-%m-%d")
|
||
today_files = [f for f in log_files if today_str in f]
|
||
|
||
if len(today_files) >= self.rotation_threshold:
|
||
message = f"日志轮转频率过高: 今天已轮转 {len(today_files)} 次 (>= {self.rotation_threshold})"
|
||
if self.alert_callback:
|
||
self.alert_callback("WARNING", message)
|
||
else:
|
||
print(f"[日志监控] WARNING: {message}")
|
||
|
||
self._last_check_time = current_time
|
||
|
||
except Exception as e:
|
||
if self.alert_callback:
|
||
self.alert_callback("ERROR", f"日志检查失败: {e}")
|
||
|
||
def _schedule_cleanup_async(self) -> None:
|
||
"""如果没有正在运行的清理任务,则在后台线程中启动一次清理。"""
|
||
# 如果未开启运行时清理功能,直接返回(清理仅限启动阶段)
|
||
if not getattr(self, "enable_runtime_cleanup", False):
|
||
return
|
||
|
||
# 用锁做“唯一运行”判定,避免竞态导致并发启动多个清理线程
|
||
with self._cleanup_lock:
|
||
if self._cleanup_running:
|
||
return
|
||
self._cleanup_running = True
|
||
|
||
keep_days, cfg_max_total_size_gb, compress_after_days, compress = (
|
||
_load_logging_cleanup_policy_from_config()
|
||
)
|
||
|
||
# 监控告警阈值与清理联动:当触发 WARNING 时,把总量上限压到 warning 阈值附近
|
||
# 这样“>=500MB 报警”能实际驱动清理生效(而不是仍按 10GB 上限导致无动作)。
|
||
try:
|
||
warning_target_gb = max(0.05, float(self.warning_size_mb) / 1024.0)
|
||
except Exception:
|
||
warning_target_gb = 0.5
|
||
|
||
effective_max_total_size_gb = min(cfg_max_total_size_gb, warning_target_gb)
|
||
|
||
def _worker():
|
||
try:
|
||
# 使用模块内的 cleanup_old_logs 执行清理,采用默认策略(保留30天、10GB上限)
|
||
try:
|
||
cleanup_old_logs(
|
||
logs_dir=self.logs_dir,
|
||
base_name=self.base_name,
|
||
keep_days=keep_days,
|
||
max_total_size_gb=effective_max_total_size_gb,
|
||
compress_after_days=compress_after_days,
|
||
compress=compress,
|
||
)
|
||
except Exception as e:
|
||
try:
|
||
if self.alert_callback:
|
||
self.alert_callback("ERROR", f"日志自动清理失败: {e}")
|
||
else:
|
||
print(f"[日志监控] 日志自动清理失败: {e}")
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
try:
|
||
with self._cleanup_lock:
|
||
self._cleanup_running = False
|
||
except Exception:
|
||
self._cleanup_running = False
|
||
|
||
t = threading.Thread(target=_worker, daemon=True)
|
||
t.start()
|
||
|
||
|
||
|
||
|
||
def start_log_monitor(
|
||
logs_dir: str,
|
||
base_name: str,
|
||
warning_size_mb: float = 500.0,
|
||
critical_size_mb: float = 2000.0,
|
||
check_interval: int = 300,
|
||
alert_callback: Optional[Callable[[str, str], None]] = None,
|
||
enable_runtime_cleanup: bool = False,
|
||
) -> None:
|
||
"""启动日志轮转监控器
|
||
|
||
参数:
|
||
logs_dir: 日志目录
|
||
base_name: 日志文件基础名称
|
||
warning_size_mb: 警告大小阈值(MB)
|
||
critical_size_mb: 严重大小阈值(MB)
|
||
check_interval: 检查间隔(秒)
|
||
alert_callback: 告警回调函数
|
||
"""
|
||
|
||
if _state._log_monitor:
|
||
_state._log_monitor.stop()
|
||
|
||
_state._log_monitor = LogRotationMonitor(
|
||
logs_dir=logs_dir,
|
||
base_name=base_name,
|
||
warning_size_mb=warning_size_mb,
|
||
critical_size_mb=critical_size_mb,
|
||
check_interval=check_interval,
|
||
alert_callback=alert_callback,
|
||
enable_runtime_cleanup=enable_runtime_cleanup,
|
||
)
|
||
_state._log_monitor.start()
|
||
|
||
|
||
|
||
|
||
def stop_log_monitor() -> None:
|
||
"""停止日志轮转监控器"""
|
||
|
||
if _state._log_monitor:
|
||
_state._log_monitor.stop()
|
||
_state._log_monitor = None
|
||
|
||
|
||
|
||
|
||
def get_log_stats(logs_dir: str, base_name: str) -> Dict[str, Any]:
|
||
"""获取日志统计信息
|
||
|
||
返回:
|
||
包含文件数量、总大小、压缩率等信息的字典
|
||
"""
|
||
stats = {
|
||
"total_files": 0,
|
||
"total_size_mb": 0.0,
|
||
"compressed_files": 0,
|
||
"compressed_size_mb": 0.0,
|
||
"compression_ratio": 0.0,
|
||
"oldest_file": None,
|
||
"newest_file": None,
|
||
}
|
||
|
||
try:
|
||
pattern = os.path.join(logs_dir, f"{base_name}_*.log*")
|
||
log_files = glob.glob(pattern)
|
||
|
||
if not log_files:
|
||
return stats
|
||
|
||
stats["total_files"] = len(log_files)
|
||
|
||
# 计算大小统计
|
||
uncompressed_size = 0
|
||
compressed_size = 0
|
||
|
||
file_times = []
|
||
for log_file in log_files:
|
||
if os.path.exists(log_file):
|
||
size = os.path.getsize(log_file)
|
||
stats["total_size_mb"] += size / 1024 / 1024
|
||
|
||
if log_file.endswith(".gz"):
|
||
stats["compressed_files"] += 1
|
||
compressed_size += size
|
||
else:
|
||
uncompressed_size += size
|
||
|
||
# 记录文件时间
|
||
try:
|
||
mtime = os.path.getmtime(log_file)
|
||
file_times.append((mtime, log_file))
|
||
except Exception:
|
||
pass
|
||
|
||
# 计算压缩率
|
||
if uncompressed_size > 0 and compressed_size > 0:
|
||
# 压缩率 = 压缩后大小 / 原始大小
|
||
stats["compression_ratio"] = compressed_size / (
|
||
uncompressed_size + compressed_size
|
||
)
|
||
|
||
# 找出最老和最新的文件
|
||
if file_times:
|
||
file_times.sort()
|
||
stats["oldest_file"] = os.path.basename(file_times[0][1])
|
||
stats["newest_file"] = os.path.basename(file_times[-1][1])
|
||
|
||
except Exception:
|
||
pass
|
||
|
||
return stats
|