- Cleaned up code formatting and improved readability in detailed_time_analysis.py. - Introduced a new script, log_filter.py, to filter log files based on user-defined keywords, with options for outputting results to a file.
208 lines
6.4 KiB
Python
208 lines
6.4 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
Detailed time analysis for logs.
|
||
Calculates Average, P80, Min, Max for various operations.
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import numpy as np
|
||
from collections import defaultdict
|
||
from datetime import datetime
|
||
|
||
# Ensure UTF-8 output on Windows
|
||
if sys.platform == "win32":
|
||
try:
|
||
if hasattr(sys.stdout, "reconfigure"):
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def parse_logs(log_paths):
|
||
data = defaultdict(list)
|
||
|
||
# 更通用的正则,适应 actions.log 和 unified log
|
||
# 匹配: [GUI] - Tag [OP] Msg ... 耗时:Val
|
||
gui_op_re = re.compile(
|
||
r"\[GUI\]\s*-\s*(.*?)\s*\[OP\]\s*(.*?)\s*(?:elapsed_seconds|elapsed_ms|耗时[:=])\s*=?\s*([\d.]+)([ms]?)",
|
||
re.I,
|
||
)
|
||
|
||
# 匹配: [Tag] Msg ... 耗时:Val (非 [GUI] 开头的格式)
|
||
tag_op_re = re.compile(
|
||
r"\[([^\]\d]{2,15})\]\s*(.*?)\s*(?:elapsed_seconds|elapsed_ms|耗时[:=])\s*=?\s*([\d.]+)([ms]?)",
|
||
re.I,
|
||
)
|
||
|
||
exclude_tags = {
|
||
"INFO",
|
||
"DEBUG",
|
||
"ERROR",
|
||
"WARN",
|
||
"WARNING",
|
||
"TRACE",
|
||
"GUI",
|
||
"ACTIONLOGGER",
|
||
}
|
||
|
||
for log_path in log_paths:
|
||
if not os.path.exists(log_path):
|
||
continue
|
||
|
||
print(f"Reading {log_path}...")
|
||
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
|
||
for line in f:
|
||
tag, msg, val_str, unit = None, None, None, None
|
||
|
||
# 尝试匹配 [GUI] - Tag [OP] 格式
|
||
m = gui_op_re.search(line)
|
||
if m:
|
||
tag, msg, val_str, unit = m.groups()
|
||
else:
|
||
# 尝试匹配 [Tag] 格式
|
||
m = tag_op_re.search(line)
|
||
if m:
|
||
tag, msg, val_str, unit = m.groups()
|
||
|
||
if tag and msg:
|
||
if tag.upper() in exclude_tags:
|
||
continue
|
||
|
||
val = float(val_str)
|
||
is_ms = False
|
||
if unit.lower() in ["ms", "m"]:
|
||
is_ms = True
|
||
elif "耗时=" in line and val > 5: # The bug
|
||
is_ms = True
|
||
|
||
value = val if is_ms else val * 1000
|
||
|
||
# 清理 Msg
|
||
# 避免使用 '.' 在正则中未转义的情况
|
||
clean_msg = msg
|
||
# 1. 移除 (等待开奖中) 这种括号内容
|
||
clean_msg = re.sub(r"(.*?等待开奖.*?)", "", clean_msg)
|
||
clean_msg = re.sub(r"\(.*?等待开奖.*?\)", "", clean_msg)
|
||
|
||
# 2. 按照特定关键词拆分,取前面的部分作为操作名
|
||
split_keywords = [
|
||
"完成",
|
||
"elapsed",
|
||
"耗时",
|
||
"成功",
|
||
"失败",
|
||
"开始",
|
||
"结束",
|
||
"...",
|
||
"---",
|
||
"步骤",
|
||
]
|
||
for kw in split_keywords:
|
||
if kw in clean_msg:
|
||
clean_msg = clean_msg.split(kw)[0]
|
||
|
||
# 3. 统一特殊长内容
|
||
if "完整识别结果" in clean_msg:
|
||
clean_msg = "完整识别结果"
|
||
if "等待开奖中" in clean_msg:
|
||
clean_msg = "等待开奖中"
|
||
if "未检测到福袋图标" in clean_msg:
|
||
clean_msg = "未检测到福袋图标"
|
||
if "检测到福袋图标" in clean_msg:
|
||
clean_msg = "检测到福袋图标"
|
||
|
||
# 4. 移除括号
|
||
clean_msg = re.sub(r"(.*?)", "", clean_msg)
|
||
clean_msg = re.sub(r"\(.*?\)", "", clean_msg)
|
||
|
||
clean_msg = clean_msg.strip(" ,,|:-_")
|
||
|
||
if not clean_msg:
|
||
continue
|
||
|
||
category = f"[{tag}] {clean_msg}"
|
||
data[category].append(value)
|
||
continue
|
||
|
||
# 特殊格式处理 (截图)
|
||
m = re.search(r"\[截图\].*?总耗时\(ms\)=([\d.]+)", line)
|
||
if m:
|
||
data["[截图] 获取屏幕截图"].append(float(m.group(1)))
|
||
continue
|
||
|
||
return data
|
||
|
||
|
||
def analyze(data):
|
||
results = []
|
||
for category, values in data.items():
|
||
if not values:
|
||
continue
|
||
arr = np.array(values)
|
||
results.append(
|
||
{
|
||
"category": category,
|
||
"count": len(arr),
|
||
"avg": np.mean(arr),
|
||
"p50": np.percentile(arr, 50),
|
||
"p80": np.percentile(arr, 80),
|
||
"p95": np.percentile(arr, 95),
|
||
"min": np.min(arr),
|
||
"max": np.max(arr),
|
||
"total": np.sum(arr),
|
||
}
|
||
)
|
||
|
||
# Sort by average duration descending
|
||
results.sort(key=lambda x: x["avg"], reverse=True)
|
||
return results
|
||
|
||
|
||
def print_table(results):
|
||
header = f"{'Operation':<55} | {'Count':>6} | {'Avg':>8} | {'P50':>8} | {'P80':>8} | {'P95':>8} | {'Max':>8}"
|
||
sep = "-" * len(header)
|
||
print(sep)
|
||
print(header)
|
||
print(sep)
|
||
for r in results:
|
||
print(
|
||
f"{r['category'][:55]:<55} | {r['count']:>6} | {r['avg']:>8.1f} | {r['p50']:>8.1f} | {r['p80']:>8.1f} | {r['p95']:>8.1f} | {r['max']:>8.1f}"
|
||
)
|
||
print(sep)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
logs_dir = "logs"
|
||
latest_unified = None
|
||
all_logs = []
|
||
|
||
# Get all logs from last 3 days or so
|
||
if os.path.exists(logs_dir):
|
||
files = os.listdir(logs_dir)
|
||
# Sort files to find the latest
|
||
files.sort(reverse=True)
|
||
|
||
# Add actions.log
|
||
if "actions.log" in files:
|
||
all_logs.append(os.path.join(logs_dir, "actions.log"))
|
||
|
||
# Add a few recent unified logs
|
||
unified_count = 0
|
||
for f in files:
|
||
if f.startswith("unified_") and f.endswith(".log"):
|
||
all_logs.append(os.path.join(logs_dir, f))
|
||
unified_count += 1
|
||
if unified_count >= 3:
|
||
break
|
||
|
||
if not all_logs:
|
||
print("No logs found.")
|
||
sys.exit(1)
|
||
|
||
data = parse_logs(all_logs)
|
||
results = analyze(data)
|
||
print_table(results)
|