Files
douyi_guaji_main/douyin/logging/douyin_logging_pkg/monitor.py
T

334 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""douyin_logging_pkg.monitor - 日志轮转 / 大小监控。
- LogRotationMonitor 轮转监控线程(按间隔检查日志大小并触发清理)
- start_log_monitor 启动监控(懒创建实例)
- stop_log_monitor 停止监控
- get_log_stats 汇总日志统计信息
"""
from __future__ import annotations
import glob
import logging
import os
import threading
import time
from datetime import datetime
from typing import Any, Callable, Dict, Optional
from douyin.core import path_config
from . import _common as _state
from .cleanup import (
_load_logging_cleanup_policy_from_config,
cleanup_old_logs,
)
class LogRotationMonitor:
"""日志轮转监控器:监控日志大小、轮转频率并发出告警"""
def __init__(
self,
logs_dir: str,
base_name: str,
warning_size_mb: float = 500.0, # 警告大小(MB)
critical_size_mb: float = 2000.0, # 严重大小(MB)
check_interval: int = 300, # 检查间隔(秒)
rotation_threshold: int = 10, # 轮转频率阈值(每天)
alert_callback: Optional[Callable[[str, str], None]] = None,
enable_runtime_cleanup: bool = False,
):
"""
参数:
logs_dir: 日志目录
base_name: 日志文件基础名称
warning_size_mb: 警告大小阈值(MB)
critical_size_mb: 严重大小阈值(MB)
check_interval: 检查间隔(秒)
rotation_threshold: 轮转频率阈值(每天)
alert_callback: 告警回调函数,参数:(level, message)
"""
self.logs_dir = logs_dir
self.base_name = base_name
self.warning_size_mb = warning_size_mb
self.critical_size_mb = critical_size_mb
self.check_interval = check_interval
self.rotation_threshold = rotation_threshold
self.alert_callback = alert_callback
self._running = False
self._thread: Optional[threading.Thread] = None
self._last_rotation_count = 0
self._last_check_time = datetime.now()
# 清理状态,避免并发触发大量清理操作
self._cleanup_lock = threading.Lock()
self._cleanup_running = False
# 是否允许在运行过程中触发自动清理(默认不允许)
self.enable_runtime_cleanup = bool(enable_runtime_cleanup)
# 告警输出限流:避免在运行时开启监控时刷屏
self._last_warning_print_ts = 0.0
self._last_critical_print_ts = 0.0
self._min_alert_print_interval_sec = 3600.0
def start(self) -> None:
"""启动监控器"""
if self._running:
return
self._running = True
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
self._thread.start()
def stop(self) -> None:
"""停止监控器"""
self._running = False
if self._thread and self._thread.is_alive():
self._thread.join(timeout=5)
def _monitor_loop(self) -> None:
"""监控循环"""
while self._running:
try:
self._check_logs()
time.sleep(self.check_interval)
except Exception as e:
if self.alert_callback:
self.alert_callback("ERROR", f"日志监控器异常: {e}")
def _check_logs(self) -> None:
"""检查日志状态"""
try:
# 收集所有日志文件
pattern = os.path.join(self.logs_dir, f"{self.base_name}_*.log*")
log_files = glob.glob(pattern)
if not log_files:
return
# 计算总大小
total_size = sum(os.path.getsize(f) for f in log_files if os.path.exists(f))
total_size_mb = total_size / 1024 / 1024
# 检查大小告警
if total_size_mb >= self.critical_size_mb:
message = f"日志总大小严重超限: {total_size_mb:.1f}MB >= {self.critical_size_mb}MB"
if self.alert_callback:
self.alert_callback("CRITICAL", message)
else:
now_ts = time.time()
if (
now_ts - self._last_critical_print_ts
>= self._min_alert_print_interval_sec
):
print(f"[日志监控] CRITICAL: {message}")
self._last_critical_print_ts = now_ts
elif total_size_mb >= self.warning_size_mb:
message = (
f"日志总大小警告: {total_size_mb:.1f}MB >= {self.warning_size_mb}MB"
)
if self.alert_callback:
self.alert_callback("WARNING", message)
else:
now_ts = time.time()
if (
now_ts - self._last_warning_print_ts
>= self._min_alert_print_interval_sec
):
print(f"[日志监控] WARNING: {message}")
self._last_warning_print_ts = now_ts
# 运行时不再自动触发清理,清理仅在启动阶段执行以避免中途 I/O 干扰
# 检查轮转频率
current_time = datetime.now()
if (
current_time - self._last_check_time
).total_seconds() >= 86400: # 24小时
# 计算今天的文件数量
today_str = current_time.strftime("%Y-%m-%d")
today_files = [f for f in log_files if today_str in f]
if len(today_files) >= self.rotation_threshold:
message = f"日志轮转频率过高: 今天已轮转 {len(today_files)} 次 (>= {self.rotation_threshold})"
if self.alert_callback:
self.alert_callback("WARNING", message)
else:
print(f"[日志监控] WARNING: {message}")
self._last_check_time = current_time
except Exception as e:
if self.alert_callback:
self.alert_callback("ERROR", f"日志检查失败: {e}")
def _schedule_cleanup_async(self) -> None:
"""如果没有正在运行的清理任务,则在后台线程中启动一次清理。"""
# 如果未开启运行时清理功能,直接返回(清理仅限启动阶段)
if not getattr(self, "enable_runtime_cleanup", False):
return
# 用锁做“唯一运行”判定,避免竞态导致并发启动多个清理线程
with self._cleanup_lock:
if self._cleanup_running:
return
self._cleanup_running = True
keep_days, cfg_max_total_size_gb, compress_after_days, compress = (
_load_logging_cleanup_policy_from_config()
)
# 监控告警阈值与清理联动:当触发 WARNING 时,把总量上限压到 warning 阈值附近
# 这样“>=500MB 报警”能实际驱动清理生效(而不是仍按 10GB 上限导致无动作)。
try:
warning_target_gb = max(0.05, float(self.warning_size_mb) / 1024.0)
except Exception:
warning_target_gb = 0.5
effective_max_total_size_gb = min(cfg_max_total_size_gb, warning_target_gb)
def _worker():
try:
# 使用模块内的 cleanup_old_logs 执行清理,采用默认策略(保留30天、10GB上限)
try:
cleanup_old_logs(
logs_dir=self.logs_dir,
base_name=self.base_name,
keep_days=keep_days,
max_total_size_gb=effective_max_total_size_gb,
compress_after_days=compress_after_days,
compress=compress,
)
except Exception as e:
try:
if self.alert_callback:
self.alert_callback("ERROR", f"日志自动清理失败: {e}")
else:
print(f"[日志监控] 日志自动清理失败: {e}")
except Exception:
pass
finally:
try:
with self._cleanup_lock:
self._cleanup_running = False
except Exception:
self._cleanup_running = False
t = threading.Thread(target=_worker, daemon=True)
t.start()
def start_log_monitor(
logs_dir: str,
base_name: str,
warning_size_mb: float = 500.0,
critical_size_mb: float = 2000.0,
check_interval: int = 300,
alert_callback: Optional[Callable[[str, str], None]] = None,
enable_runtime_cleanup: bool = False,
) -> None:
"""启动日志轮转监控器
参数:
logs_dir: 日志目录
base_name: 日志文件基础名称
warning_size_mb: 警告大小阈值(MB)
critical_size_mb: 严重大小阈值(MB)
check_interval: 检查间隔(秒)
alert_callback: 告警回调函数
"""
if _state._log_monitor:
_state._log_monitor.stop()
_state._log_monitor = LogRotationMonitor(
logs_dir=logs_dir,
base_name=base_name,
warning_size_mb=warning_size_mb,
critical_size_mb=critical_size_mb,
check_interval=check_interval,
alert_callback=alert_callback,
enable_runtime_cleanup=enable_runtime_cleanup,
)
_state._log_monitor.start()
def stop_log_monitor() -> None:
"""停止日志轮转监控器"""
if _state._log_monitor:
_state._log_monitor.stop()
_state._log_monitor = None
def get_log_stats(logs_dir: str, base_name: str) -> Dict[str, Any]:
"""获取日志统计信息
返回:
包含文件数量、总大小、压缩率等信息的字典
"""
stats = {
"total_files": 0,
"total_size_mb": 0.0,
"compressed_files": 0,
"compressed_size_mb": 0.0,
"compression_ratio": 0.0,
"oldest_file": None,
"newest_file": None,
}
try:
pattern = os.path.join(logs_dir, f"{base_name}_*.log*")
log_files = glob.glob(pattern)
if not log_files:
return stats
stats["total_files"] = len(log_files)
# 计算大小统计
uncompressed_size = 0
compressed_size = 0
file_times = []
for log_file in log_files:
if os.path.exists(log_file):
size = os.path.getsize(log_file)
stats["total_size_mb"] += size / 1024 / 1024
if log_file.endswith(".gz"):
stats["compressed_files"] += 1
compressed_size += size
else:
uncompressed_size += size
# 记录文件时间
try:
mtime = os.path.getmtime(log_file)
file_times.append((mtime, log_file))
except Exception:
pass
# 计算压缩率
if uncompressed_size > 0 and compressed_size > 0:
# 压缩率 = 压缩后大小 / 原始大小
stats["compression_ratio"] = compressed_size / (
uncompressed_size + compressed_size
)
# 找出最老和最新的文件
if file_times:
file_times.sort()
stats["oldest_file"] = os.path.basename(file_times[0][1])
stats["newest_file"] = os.path.basename(file_times[-1][1])
except Exception:
pass
return stats