90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Evaluate HLSQConvNet checkpoint on validation images."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
|
|
from hls_qcnn import create_hls_qcnn_from_checkpoint
|
|
from train_hls_qcnn import HLSImageDataset, configure_stdio
|
|
|
|
|
|
configure_stdio()
|
|
|
|
|
|
@torch.no_grad()
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Evaluate HLS-friendly CNN")
|
|
parser.add_argument("--checkpoint", default="../data/models/hls_qcnn_best.pth")
|
|
parser.add_argument("--data_dir", default="../data/processed/val")
|
|
parser.add_argument("--output", default="../results/hls_qcnn_eval.json")
|
|
parser.add_argument("--min_acc", type=float, default=70.0)
|
|
args = parser.parse_args()
|
|
|
|
checkpoint_path = Path(args.checkpoint)
|
|
if not checkpoint_path.exists():
|
|
print(f"Checkpoint not found: {checkpoint_path}")
|
|
return 2
|
|
|
|
checkpoint = torch.load(checkpoint_path, map_location="cpu")
|
|
model = create_hls_qcnn_from_checkpoint(checkpoint)
|
|
model.load_state_dict(checkpoint["model_state_dict"])
|
|
model.eval()
|
|
|
|
input_size = int(checkpoint.get("model_config", {}).get("input_size", 128))
|
|
dataset = HLSImageDataset(args.data_dir, input_size=input_size, augment=False)
|
|
if len(dataset) == 0:
|
|
print(f"No images found: {args.data_dir}")
|
|
return 2
|
|
|
|
correct = 0
|
|
class_correct = [0] * 10
|
|
class_total = [0] * 10
|
|
latencies = []
|
|
for image, label in dataset:
|
|
start = time.perf_counter()
|
|
logits = model(image.unsqueeze(0))
|
|
latencies.append(time.perf_counter() - start)
|
|
pred = int(logits.argmax(dim=1).item())
|
|
correct += int(pred == label)
|
|
class_total[label] += 1
|
|
if pred == label:
|
|
class_correct[label] += 1
|
|
|
|
accuracy = 100.0 * correct / len(dataset)
|
|
avg_latency = sum(latencies) / len(latencies)
|
|
fps = 1.0 / avg_latency if avg_latency > 0 else 0.0
|
|
metrics = {
|
|
"checkpoint": str(checkpoint_path),
|
|
"num_images": len(dataset),
|
|
"correct": correct,
|
|
"accuracy": accuracy,
|
|
"class_correct": class_correct,
|
|
"class_total": class_total,
|
|
"avg_latency_ms": avg_latency * 1000.0,
|
|
"fps": fps,
|
|
"model_config": checkpoint.get("model_config"),
|
|
}
|
|
|
|
output_path = Path(args.output)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with output_path.open("w", encoding="utf-8") as f:
|
|
json.dump(metrics, f, indent=2, ensure_ascii=False)
|
|
|
|
print(f"Accuracy: {accuracy:.2f}% ({correct}/{len(dataset)})")
|
|
print(f"Average CPU latency: {avg_latency * 1000.0:.2f} ms, FPS: {fps:.2f}")
|
|
print(f"Metrics written: {output_path}")
|
|
if accuracy < args.min_acc:
|
|
print(f"Accuracy gate failed: {accuracy:.2f}% < {args.min_acc:.2f}%")
|
|
return 2
|
|
print("Accuracy gate passed.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|