Files
douyi_guaji_main/tools/detailed_time_analysis.py
T
Your Name d1f1c8f682 Refactor detailed_time_analysis.py for improved readability and add log_filter.py for keyword-based log filtering
- Cleaned up code formatting and improved readability in detailed_time_analysis.py.
- Introduced a new script, log_filter.py, to filter log files based on user-defined keywords, with options for outputting results to a file.
2026-01-12 23:48:10 +08:00

208 lines
6.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Detailed time analysis for logs.
Calculates Average, P80, Min, Max for various operations.
"""
import os
import re
import sys
import numpy as np
from collections import defaultdict
from datetime import datetime
# Ensure UTF-8 output on Windows
if sys.platform == "win32":
try:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
except Exception:
pass
def parse_logs(log_paths):
data = defaultdict(list)
# 更通用的正则,适应 actions.log 和 unified log
# 匹配: [GUI] - Tag [OP] Msg ... 耗时:Val
gui_op_re = re.compile(
r"\[GUI\]\s*-\s*(.*?)\s*\[OP\]\s*(.*?)\s*(?:elapsed_seconds|elapsed_ms|耗时[:=])\s*=?\s*([\d.]+)([ms]?)",
re.I,
)
# 匹配: [Tag] Msg ... 耗时:Val (非 [GUI] 开头的格式)
tag_op_re = re.compile(
r"\[([^\]\d]{2,15})\]\s*(.*?)\s*(?:elapsed_seconds|elapsed_ms|耗时[:=])\s*=?\s*([\d.]+)([ms]?)",
re.I,
)
exclude_tags = {
"INFO",
"DEBUG",
"ERROR",
"WARN",
"WARNING",
"TRACE",
"GUI",
"ACTIONLOGGER",
}
for log_path in log_paths:
if not os.path.exists(log_path):
continue
print(f"Reading {log_path}...")
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
tag, msg, val_str, unit = None, None, None, None
# 尝试匹配 [GUI] - Tag [OP] 格式
m = gui_op_re.search(line)
if m:
tag, msg, val_str, unit = m.groups()
else:
# 尝试匹配 [Tag] 格式
m = tag_op_re.search(line)
if m:
tag, msg, val_str, unit = m.groups()
if tag and msg:
if tag.upper() in exclude_tags:
continue
val = float(val_str)
is_ms = False
if unit.lower() in ["ms", "m"]:
is_ms = True
elif "耗时=" in line and val > 5: # The bug
is_ms = True
value = val if is_ms else val * 1000
# 清理 Msg
# 避免使用 '.' 在正则中未转义的情况
clean_msg = msg
# 1. 移除 (等待开奖中) 这种括号内容
clean_msg = re.sub(r"(.*?等待开奖.*?)", "", clean_msg)
clean_msg = re.sub(r"\(.*?等待开奖.*?\)", "", clean_msg)
# 2. 按照特定关键词拆分,取前面的部分作为操作名
split_keywords = [
"完成",
"elapsed",
"耗时",
"成功",
"失败",
"开始",
"结束",
"...",
"---",
"步骤",
]
for kw in split_keywords:
if kw in clean_msg:
clean_msg = clean_msg.split(kw)[0]
# 3. 统一特殊长内容
if "完整识别结果" in clean_msg:
clean_msg = "完整识别结果"
if "等待开奖中" in clean_msg:
clean_msg = "等待开奖中"
if "未检测到福袋图标" in clean_msg:
clean_msg = "未检测到福袋图标"
if "检测到福袋图标" in clean_msg:
clean_msg = "检测到福袋图标"
# 4. 移除括号
clean_msg = re.sub(r"(.*?)", "", clean_msg)
clean_msg = re.sub(r"\(.*?\)", "", clean_msg)
clean_msg = clean_msg.strip(" ,,|:-_")
if not clean_msg:
continue
category = f"[{tag}] {clean_msg}"
data[category].append(value)
continue
# 特殊格式处理 (截图)
m = re.search(r"\[截图\].*?总耗时\(ms\)=([\d.]+)", line)
if m:
data["[截图] 获取屏幕截图"].append(float(m.group(1)))
continue
return data
def analyze(data):
results = []
for category, values in data.items():
if not values:
continue
arr = np.array(values)
results.append(
{
"category": category,
"count": len(arr),
"avg": np.mean(arr),
"p50": np.percentile(arr, 50),
"p80": np.percentile(arr, 80),
"p95": np.percentile(arr, 95),
"min": np.min(arr),
"max": np.max(arr),
"total": np.sum(arr),
}
)
# Sort by average duration descending
results.sort(key=lambda x: x["avg"], reverse=True)
return results
def print_table(results):
header = f"{'Operation':<55} | {'Count':>6} | {'Avg':>8} | {'P50':>8} | {'P80':>8} | {'P95':>8} | {'Max':>8}"
sep = "-" * len(header)
print(sep)
print(header)
print(sep)
for r in results:
print(
f"{r['category'][:55]:<55} | {r['count']:>6} | {r['avg']:>8.1f} | {r['p50']:>8.1f} | {r['p80']:>8.1f} | {r['p95']:>8.1f} | {r['max']:>8.1f}"
)
print(sep)
if __name__ == "__main__":
logs_dir = "logs"
latest_unified = None
all_logs = []
# Get all logs from last 3 days or so
if os.path.exists(logs_dir):
files = os.listdir(logs_dir)
# Sort files to find the latest
files.sort(reverse=True)
# Add actions.log
if "actions.log" in files:
all_logs.append(os.path.join(logs_dir, "actions.log"))
# Add a few recent unified logs
unified_count = 0
for f in files:
if f.startswith("unified_") and f.endswith(".log"):
all_logs.append(os.path.join(logs_dir, f))
unified_count += 1
if unified_count >= 3:
break
if not all_logs:
print("No logs found.")
sys.exit(1)
data = parse_logs(all_logs)
results = analyze(data)
print_table(results)