Files
douyi_guaji_main/douyin/logging/douyin_logging_pkg/cleanup.py
T

285 lines
8.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""douyin_logging_pkg.cleanup - 日志压缩与清理策略。
- compress_log_file 压缩单个日志文件为 .gz
- cleanup_old_logs 按大小 / 数量 / 天数三重清理
- _load_logging_cleanup_policy_from_config 从 config.json 读取清理策略
"""
from __future__ import annotations
import glob
import gzip
import logging
import os
import shutil
import time
from datetime import datetime, timedelta
from typing import Any, Dict
from douyin.core import path_config
def compress_log_file(log_file: str) -> bool:
"""压缩单个日志文件
参数:
log_file: 日志文件路径
返回:
是否压缩成功
"""
try:
if not os.path.exists(log_file):
return False
# 检查文件大小,只有较大文件才压缩
file_size = os.path.getsize(log_file)
if file_size < 1024 * 1024: # 小于1MB不压缩
return False
compressed_file = log_file + ".gz"
# 如果压缩文件已存在,跳过
if os.path.exists(compressed_file):
return False
# 压缩文件
with open(log_file, "rb") as f_in:
with gzip.open(compressed_file, "wb", compresslevel=9) as f_out:
shutil.copyfileobj(f_in, f_out)
# 验证压缩文件
if os.path.exists(compressed_file):
compressed_size = os.path.getsize(compressed_file)
if compressed_size > 0:
# 删除原文件
os.remove(log_file)
try:
print(
f"[日志压缩] 压缩完成: {os.path.basename(log_file)} "
f"({file_size/1024:.1f}KB -> {compressed_size/1024:.1f}KB, "
f"压缩率: {compressed_size/file_size*100:.1f}%)"
)
except Exception:
pass
return True
return False
except Exception as e:
try:
print(f"[日志压缩] 压缩失败 {log_file}: {e}")
except Exception:
pass
return False
def cleanup_old_logs(
logs_dir: str,
base_name: str,
keep_days: int = 30,
max_total_size_gb: float = 10.0, # 最大总大小(GB)
compress_after_days: int = 7, # 几天后压缩
compress: bool = True,
) -> Dict[str, int]:
"""智能清理旧日志文件:支持时间、大小、压缩三重策略
参数:
logs_dir: 日志目录路径
base_name: 日志文件基础名称(不含日期和扩展名),如 "unified"
keep_days: 保留天数,默认30天
max_total_size_gb: 最大总大小(GB),默认10GB
compress_after_days: 几天后压缩,默认7天
compress: 是否启用压缩,默认True
返回:
清理统计信息
"""
stats = {
"deleted": 0,
"compressed": 0,
"total_size_mb": 0.0,
"freed_space_mb": 0.0,
}
def _safe_get_mtime(path: str) -> float:
try:
return os.path.getmtime(path)
except Exception:
return 0.0
def _safe_get_size(path: str) -> int:
try:
return os.path.getsize(path)
except Exception:
return 0
try:
if not os.path.exists(logs_dir):
return stats
# 收集所有相关日志文件(包括压缩文件)
patterns = [
os.path.join(logs_dir, f"{base_name}_*.log"),
os.path.join(logs_dir, f"{base_name}_*.log.gz"),
]
all_files: list[str] = []
for pattern in patterns:
for p in glob.glob(pattern):
# 并发轮转/删除时避免后续排序抛 FileNotFoundError
if os.path.exists(p):
all_files.append(p)
if not all_files:
return stats
# 按修改时间排序(最新的在前)
all_files.sort(key=_safe_get_mtime, reverse=True)
# 计算总大小
total_size = sum(_safe_get_size(f) for f in all_files if os.path.exists(f))
stats["total_size_mb"] = total_size / 1024 / 1024
# 如果总大小超过限制,先删除最旧的文件
max_total_size_bytes = max_total_size_gb * 1024 * 1024 * 1024
if total_size > max_total_size_bytes:
# 按时间排序,删除最旧的
sorted_files = sorted(
[f for f in all_files if os.path.exists(f)],
key=_safe_get_mtime,
)
for log_file in sorted_files:
if total_size <= max_total_size_bytes:
break
try:
file_size = _safe_get_size(log_file)
os.remove(log_file)
total_size -= file_size
stats["deleted"] += 1
stats["freed_space_mb"] += file_size / 1024 / 1024
except Exception as e:
try:
print(f"[日志清理] 删除文件失败 {log_file}: {e}")
except Exception:
pass
# 处理每个文件:压缩或删除
cutoff_date = datetime.now() - timedelta(days=keep_days)
compress_date = datetime.now() - timedelta(days=compress_after_days)
for log_file in all_files:
try:
if not os.path.exists(log_file):
continue
# 从文件名提取日期
filename = os.path.basename(log_file)
# 处理压缩文件扩展名
if filename.endswith(".gz"):
filename = filename[:-3]
# 提取日期部分
date_part = filename.replace(f"{base_name}_", "").split(".")[0]
try:
file_date = datetime.strptime(date_part, "%Y-%m-%d")
except ValueError:
continue
# 如果超过保留天数,删除
if file_date < cutoff_date:
file_size = os.path.getsize(log_file)
os.remove(log_file)
stats["deleted"] += 1
stats["freed_space_mb"] += file_size / 1024 / 1024
# 如果超过压缩天数且未压缩,压缩
elif (
compress
and file_date < compress_date
and not log_file.endswith(".gz")
):
if compress_log_file(log_file):
stats["compressed"] += 1
except Exception as e:
try:
print(f"[日志清理] 处理文件失败 {log_file}: {e}")
except Exception:
pass
# 输出统计信息
if stats["deleted"] > 0 or stats["compressed"] > 0:
try:
print(
f"[日志清理] 完成 - 删除:{stats['deleted']}, 压缩:{stats['compressed']}, "
f"释放:{stats['freed_space_mb']:.1f}MB"
)
except Exception:
pass
except Exception as e:
try:
print(f"[日志清理] 清理过程出错: {e}")
except Exception:
pass
return stats
def _load_logging_cleanup_policy_from_config() -> tuple[int, float, int, bool]:
"""从 config/config.json 读取日志清理策略。
返回:
(keep_days, max_total_size_gb, compress_after_days, compress)
说明:
- 只负责读取与容错;不做“阈值联动”的业务决策。
- 读取失败时回退到模块默认值。
"""
keep_days = 30
max_total_size_gb = 10.0
compress_after_days = 7
compress = True
try:
import json
config_path = path_config.get_config_path()
if not os.path.exists(config_path):
return keep_days, max_total_size_gb, compress_after_days, compress
with open(config_path, "r", encoding="utf-8") as f:
cfg = json.load(f)
log_cfg = cfg.get("logging", {})
if isinstance(log_cfg, dict):
keep_days = int(log_cfg.get("keep_days", keep_days))
max_total_size_gb = float(
log_cfg.get("max_total_size_gb", max_total_size_gb)
)
compress_after_days = int(
log_cfg.get("compress_after_days", compress_after_days)
)
compress = bool(log_cfg.get("compress_logs", compress))
except Exception:
pass
# 基础兜底,避免异常配置导致“全部删光/不清理”
if keep_days < 0:
keep_days = 0
if compress_after_days < 0:
compress_after_days = 0
if max_total_size_gb <= 0:
# 若用户配置为 0/负数,视为禁用“按总量限制”,回退到默认
max_total_size_gb = 10.0
return keep_days, max_total_size_gb, compress_after_days, compress