285 lines
8.9 KiB
Python
285 lines
8.9 KiB
Python
"""douyin_logging_pkg.cleanup - 日志压缩与清理策略。
|
||
|
||
- compress_log_file 压缩单个日志文件为 .gz
|
||
- cleanup_old_logs 按大小 / 数量 / 天数三重清理
|
||
- _load_logging_cleanup_policy_from_config 从 config.json 读取清理策略
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import glob
|
||
import gzip
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import time
|
||
from datetime import datetime, timedelta
|
||
from typing import Any, Dict
|
||
|
||
from douyin.core import path_config
|
||
|
||
|
||
def compress_log_file(log_file: str) -> bool:
|
||
"""压缩单个日志文件
|
||
|
||
参数:
|
||
log_file: 日志文件路径
|
||
|
||
返回:
|
||
是否压缩成功
|
||
"""
|
||
try:
|
||
if not os.path.exists(log_file):
|
||
return False
|
||
|
||
# 检查文件大小,只有较大文件才压缩
|
||
file_size = os.path.getsize(log_file)
|
||
if file_size < 1024 * 1024: # 小于1MB不压缩
|
||
return False
|
||
|
||
compressed_file = log_file + ".gz"
|
||
|
||
# 如果压缩文件已存在,跳过
|
||
if os.path.exists(compressed_file):
|
||
return False
|
||
|
||
# 压缩文件
|
||
with open(log_file, "rb") as f_in:
|
||
with gzip.open(compressed_file, "wb", compresslevel=9) as f_out:
|
||
shutil.copyfileobj(f_in, f_out)
|
||
|
||
# 验证压缩文件
|
||
if os.path.exists(compressed_file):
|
||
compressed_size = os.path.getsize(compressed_file)
|
||
if compressed_size > 0:
|
||
# 删除原文件
|
||
os.remove(log_file)
|
||
try:
|
||
print(
|
||
f"[日志压缩] 压缩完成: {os.path.basename(log_file)} "
|
||
f"({file_size/1024:.1f}KB -> {compressed_size/1024:.1f}KB, "
|
||
f"压缩率: {compressed_size/file_size*100:.1f}%)"
|
||
)
|
||
except Exception:
|
||
pass
|
||
return True
|
||
|
||
return False
|
||
except Exception as e:
|
||
try:
|
||
print(f"[日志压缩] 压缩失败 {log_file}: {e}")
|
||
except Exception:
|
||
pass
|
||
return False
|
||
|
||
|
||
|
||
|
||
def cleanup_old_logs(
|
||
logs_dir: str,
|
||
base_name: str,
|
||
keep_days: int = 30,
|
||
max_total_size_gb: float = 10.0, # 最大总大小(GB)
|
||
compress_after_days: int = 7, # 几天后压缩
|
||
compress: bool = True,
|
||
) -> Dict[str, int]:
|
||
"""智能清理旧日志文件:支持时间、大小、压缩三重策略
|
||
|
||
参数:
|
||
logs_dir: 日志目录路径
|
||
base_name: 日志文件基础名称(不含日期和扩展名),如 "unified"
|
||
keep_days: 保留天数,默认30天
|
||
max_total_size_gb: 最大总大小(GB),默认10GB
|
||
compress_after_days: 几天后压缩,默认7天
|
||
compress: 是否启用压缩,默认True
|
||
|
||
返回:
|
||
清理统计信息
|
||
"""
|
||
stats = {
|
||
"deleted": 0,
|
||
"compressed": 0,
|
||
"total_size_mb": 0.0,
|
||
"freed_space_mb": 0.0,
|
||
}
|
||
|
||
def _safe_get_mtime(path: str) -> float:
|
||
try:
|
||
return os.path.getmtime(path)
|
||
except Exception:
|
||
return 0.0
|
||
|
||
def _safe_get_size(path: str) -> int:
|
||
try:
|
||
return os.path.getsize(path)
|
||
except Exception:
|
||
return 0
|
||
|
||
try:
|
||
if not os.path.exists(logs_dir):
|
||
return stats
|
||
|
||
# 收集所有相关日志文件(包括压缩文件)
|
||
patterns = [
|
||
os.path.join(logs_dir, f"{base_name}_*.log"),
|
||
os.path.join(logs_dir, f"{base_name}_*.log.gz"),
|
||
]
|
||
|
||
all_files: list[str] = []
|
||
for pattern in patterns:
|
||
for p in glob.glob(pattern):
|
||
# 并发轮转/删除时避免后续排序抛 FileNotFoundError
|
||
if os.path.exists(p):
|
||
all_files.append(p)
|
||
|
||
if not all_files:
|
||
return stats
|
||
|
||
# 按修改时间排序(最新的在前)
|
||
all_files.sort(key=_safe_get_mtime, reverse=True)
|
||
|
||
# 计算总大小
|
||
total_size = sum(_safe_get_size(f) for f in all_files if os.path.exists(f))
|
||
stats["total_size_mb"] = total_size / 1024 / 1024
|
||
|
||
# 如果总大小超过限制,先删除最旧的文件
|
||
max_total_size_bytes = max_total_size_gb * 1024 * 1024 * 1024
|
||
if total_size > max_total_size_bytes:
|
||
# 按时间排序,删除最旧的
|
||
sorted_files = sorted(
|
||
[f for f in all_files if os.path.exists(f)],
|
||
key=_safe_get_mtime,
|
||
)
|
||
|
||
for log_file in sorted_files:
|
||
if total_size <= max_total_size_bytes:
|
||
break
|
||
|
||
try:
|
||
file_size = _safe_get_size(log_file)
|
||
os.remove(log_file)
|
||
total_size -= file_size
|
||
stats["deleted"] += 1
|
||
stats["freed_space_mb"] += file_size / 1024 / 1024
|
||
except Exception as e:
|
||
try:
|
||
print(f"[日志清理] 删除文件失败 {log_file}: {e}")
|
||
except Exception:
|
||
pass
|
||
|
||
# 处理每个文件:压缩或删除
|
||
cutoff_date = datetime.now() - timedelta(days=keep_days)
|
||
compress_date = datetime.now() - timedelta(days=compress_after_days)
|
||
|
||
for log_file in all_files:
|
||
try:
|
||
if not os.path.exists(log_file):
|
||
continue
|
||
|
||
# 从文件名提取日期
|
||
filename = os.path.basename(log_file)
|
||
# 处理压缩文件扩展名
|
||
if filename.endswith(".gz"):
|
||
filename = filename[:-3]
|
||
|
||
# 提取日期部分
|
||
date_part = filename.replace(f"{base_name}_", "").split(".")[0]
|
||
|
||
try:
|
||
file_date = datetime.strptime(date_part, "%Y-%m-%d")
|
||
except ValueError:
|
||
continue
|
||
|
||
# 如果超过保留天数,删除
|
||
if file_date < cutoff_date:
|
||
file_size = os.path.getsize(log_file)
|
||
os.remove(log_file)
|
||
stats["deleted"] += 1
|
||
stats["freed_space_mb"] += file_size / 1024 / 1024
|
||
|
||
# 如果超过压缩天数且未压缩,压缩
|
||
elif (
|
||
compress
|
||
and file_date < compress_date
|
||
and not log_file.endswith(".gz")
|
||
):
|
||
if compress_log_file(log_file):
|
||
stats["compressed"] += 1
|
||
|
||
except Exception as e:
|
||
try:
|
||
print(f"[日志清理] 处理文件失败 {log_file}: {e}")
|
||
except Exception:
|
||
pass
|
||
|
||
# 输出统计信息
|
||
if stats["deleted"] > 0 or stats["compressed"] > 0:
|
||
try:
|
||
print(
|
||
f"[日志清理] 完成 - 删除:{stats['deleted']}, 压缩:{stats['compressed']}, "
|
||
f"释放:{stats['freed_space_mb']:.1f}MB"
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
except Exception as e:
|
||
try:
|
||
print(f"[日志清理] 清理过程出错: {e}")
|
||
except Exception:
|
||
pass
|
||
|
||
return stats
|
||
|
||
|
||
|
||
|
||
def _load_logging_cleanup_policy_from_config() -> tuple[int, float, int, bool]:
|
||
"""从 config/config.json 读取日志清理策略。
|
||
|
||
返回:
|
||
(keep_days, max_total_size_gb, compress_after_days, compress)
|
||
|
||
说明:
|
||
- 只负责读取与容错;不做“阈值联动”的业务决策。
|
||
- 读取失败时回退到模块默认值。
|
||
"""
|
||
keep_days = 30
|
||
max_total_size_gb = 10.0
|
||
compress_after_days = 7
|
||
compress = True
|
||
|
||
try:
|
||
import json
|
||
|
||
config_path = path_config.get_config_path()
|
||
if not os.path.exists(config_path):
|
||
return keep_days, max_total_size_gb, compress_after_days, compress
|
||
|
||
with open(config_path, "r", encoding="utf-8") as f:
|
||
cfg = json.load(f)
|
||
|
||
log_cfg = cfg.get("logging", {})
|
||
if isinstance(log_cfg, dict):
|
||
keep_days = int(log_cfg.get("keep_days", keep_days))
|
||
max_total_size_gb = float(
|
||
log_cfg.get("max_total_size_gb", max_total_size_gb)
|
||
)
|
||
compress_after_days = int(
|
||
log_cfg.get("compress_after_days", compress_after_days)
|
||
)
|
||
compress = bool(log_cfg.get("compress_logs", compress))
|
||
except Exception:
|
||
pass
|
||
|
||
# 基础兜底,避免异常配置导致“全部删光/不清理”
|
||
if keep_days < 0:
|
||
keep_days = 0
|
||
if compress_after_days < 0:
|
||
compress_after_days = 0
|
||
if max_total_size_gb <= 0:
|
||
# 若用户配置为 0/负数,视为禁用“按总量限制”,回退到默认
|
||
max_total_size_gb = 10.0
|
||
|
||
return keep_days, max_total_size_gb, compress_after_days, compress
|
||
|
||
|