Files

92 lines
2.9 KiB
Python

"""Evaluate a trained FINN QCNN checkpoint on the validation set."""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
import torch
from finn_qcnn import create_finn_qcnn_from_checkpoint
from train_finn_qcnn import FINNImageDataset, configure_stdio
configure_stdio()
@torch.no_grad()
def main() -> int:
parser = argparse.ArgumentParser(description="Evaluate FINN QCNN checkpoint")
parser.add_argument("--checkpoint", default="../data/models/finn_qcnn_best.pth")
parser.add_argument("--data_dir", default="../data/processed/val")
parser.add_argument("--output", default="../results/finn_qcnn_eval.json")
parser.add_argument("--min_acc", type=float, default=70.0)
args = parser.parse_args()
checkpoint_path = Path(args.checkpoint)
if not checkpoint_path.exists():
print(f"Checkpoint not found: {checkpoint_path}")
return 2
checkpoint = torch.load(checkpoint_path, map_location="cpu")
model = create_finn_qcnn_from_checkpoint(checkpoint)
model.load_state_dict(checkpoint["model_state_dict"])
model.eval()
input_size = int(checkpoint.get("model_config", {}).get("input_size", 128))
dataset = FINNImageDataset(args.data_dir, input_size=input_size, augment=False)
if len(dataset) == 0:
print(f"No validation images found: {args.data_dir}")
return 2
correct = 0
class_correct = [0] * 10
class_total = [0] * 10
latencies = []
for image, label in dataset:
start = time.perf_counter()
logits = model(image.unsqueeze(0))
latencies.append(time.perf_counter() - start)
pred = int(logits.argmax(dim=1).item())
correct += int(pred == label)
class_total[label] += 1
if pred == label:
class_correct[label] += 1
accuracy = 100.0 * correct / len(dataset)
avg_latency = sum(latencies) / len(latencies)
fps = 1.0 / avg_latency if avg_latency > 0 else 0.0
metrics = {
"checkpoint": str(checkpoint_path),
"num_images": len(dataset),
"accuracy": accuracy,
"correct": correct,
"class_correct": class_correct,
"class_total": class_total,
"avg_latency_ms": avg_latency * 1000.0,
"fps": fps,
"model_config": checkpoint.get("model_config"),
}
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
with output_path.open("w", encoding="utf-8") as f:
json.dump(metrics, f, indent=2, ensure_ascii=False)
print(f"Accuracy: {accuracy:.2f}% ({correct}/{len(dataset)})")
print(f"Average CPU latency: {avg_latency * 1000.0:.2f} ms, FPS: {fps:.2f}")
print(f"Metrics written: {output_path}")
if accuracy < args.min_acc:
print(f"Accuracy gate failed: {accuracy:.2f}% < {args.min_acc:.2f}%")
return 2
print("Accuracy gate passed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())