Files
dxdesigner-part-replacement/tests/part_replace/test_controller.py
T

2135 lines
83 KiB
Python

from __future__ import annotations
from dataclasses import replace
from decimal import Decimal
import hashlib
import json
from pathlib import Path
import pytest
import dx_productivity.part_replace.controller as controller_module
from dx_productivity.part_replace.certification import CertificationStore, interface_fingerprint
from dx_productivity.part_replace.command_schema import command_from_data
from dx_productivity.part_replace.controller import BatchController, ControllerError
from dx_productivity.part_replace.inputs import parse_mapping_bytes, parse_projects_bytes
from dx_productivity.part_replace.models import (
ComponentSnapshot,
DesignSnapshot,
ExternalChecks,
MappingRule,
PinSnapshot,
ProjectState,
UnitSnapshot,
)
from dx_productivity.part_replace.protocol import (
BridgeCommand,
atomic_write_text,
snapshot_fingerprint,
write_snapshot,
)
from dx_productivity.part_replace.run_store import RunStore, StateError
from dx_productivity.part_replace.verification import verify_replacement
def _component(part_number: str, *, symbol: str = "LIB:OLD", net: str = "N1") -> ComponentSnapshot:
return ComponentSnapshot(
logical_key="/MAIN|U1",
refdes="U1",
part_number=part_number,
attributes=(("manufacturer", "Acme"),),
units=(
UnitSnapshot(
unit_id="A",
sheet="/MAIN",
symbol=symbol,
origin_x=Decimal("9.5"),
origin_y=Decimal("19.5"),
x=Decimal("10"),
y=Decimal("20"),
rotation=0,
mirrored=False,
pins=(
PinSnapshot("1", "IN", Decimal("10.25"), Decimal("20"), net),
PinSnapshot("2", "OUT", Decimal("10.5"), Decimal("20"), "N2"),
),
),
),
)
def _snapshot(project_id: str, project_path: Path, part_number: str, fingerprint: str, **component_changes: str) -> DesignSnapshot:
return DesignSnapshot(
format_version=2,
project_id=project_id,
project_path=str(project_path.resolve()),
fingerprint=fingerprint,
components=(_component(part_number, **component_changes),),
)
def _write_inputs(tmp_path: Path, project_count: int = 1) -> tuple[Path, Path, tuple[Path, ...]]:
mapping = tmp_path / "mapping.csv"
mapping.write_text("old_part_number,new_part_number\nOLD-1,NEW-1\n", encoding="utf-8")
markers: list[Path] = []
for index in range(1, project_count + 1):
project_root = tmp_path / "projects" / f"p{index}"
project_root.mkdir(parents=True)
marker = project_root / f"p{index}.prj"
marker.write_text(f"project {index}\n", encoding="utf-8")
markers.append(marker)
projects = tmp_path / "projects.txt"
projects.write_text("".join(f"{marker.resolve()}\n" for marker in markers), encoding="utf-8")
return mapping, projects, tuple(markers)
def _certify(path: Path, before: DesignSnapshot, *, target_symbol: str = "LIB:NEW") -> None:
rule = MappingRule("OLD-1", "NEW-1")
after = replace(
before,
fingerprint="trial-after",
components=(_component("NEW-1", symbol=target_symbol),),
)
result = verify_replacement(before, after, rule, ("/MAIN|U1",))
store = CertificationStore(
path,
before=before,
after=after,
verification=result,
evidence_paths=("before.json", "after.json", "packager.txt", "erc.txt", "netlist.txt"),
)
store.record(
rule,
interface_fingerprint(after, ("/MAIN|U1",)),
"lib-r42",
ExternalChecks(packager=True, erc=True, netlist=True),
)
def _record_certification(
path: Path,
before: DesignSnapshot,
after: DesignSnapshot,
rule: MappingRule,
logical_key: str,
) -> None:
result = verify_replacement(before, after, rule, (logical_key,))
CertificationStore(
path,
before=before,
after=after,
verification=result,
evidence_paths=(
"before.json",
"after.json",
"packager.txt",
"erc.txt",
"netlist.txt",
),
).record(
rule,
interface_fingerprint(after, (logical_key,)),
"lib-r42",
ExternalChecks(packager=True, erc=True, netlist=True),
)
def _create_controller(tmp_path: Path, project_count: int = 1) -> tuple[BatchController, tuple[Path, ...], Path]:
mapping, projects, markers = _write_inputs(tmp_path, project_count)
certification_path = tmp_path / "certifications.json"
controller = BatchController.create_job(
tmp_path / "run",
mapping,
projects,
certification_path=certification_path,
allowed_suffixes=(".prj",),
library_identity="lib-r42",
)
return controller, markers, certification_path
def _publish_bridge_snapshot(
controller: BatchController,
command: BridgeCommand,
state: str,
snapshot: DesignSnapshot,
*,
status_command: BridgeCommand | None = None,
) -> tuple[DesignSnapshot, Path]:
correlated = command if status_command is None else status_command
observed = replace(snapshot, fingerprint=snapshot_fingerprint(snapshot))
mailbox = controller.command_path(command.project_id).parent
snapshot_path = mailbox / "snapshot.json"
write_snapshot(snapshot_path, observed)
atomic_write_text(
mailbox / "status.json",
json.dumps(
{
"command_id": correlated.command_id,
"details": [
["action", correlated.action],
["expected_count", str(correlated.expected_count)],
["job_id", correlated.job_id],
],
"error_code": None,
"error_message": None,
"format_version": 1,
"project_id": correlated.project_id,
"snapshot_path": str(snapshot_path.resolve()),
"state": state,
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
+ "\n",
)
return observed, snapshot_path
def _publish_waiting_status(
controller: BatchController,
command: BridgeCommand,
state: str,
*,
manual_route: str | None = None,
) -> None:
details = [
["action", command.action],
["expected_count", str(command.expected_count)],
["instruction", "complete the current native action"],
["job_id", command.job_id],
]
if manual_route is not None:
details.append(["manual_route", manual_route])
atomic_write_text(
controller.command_path(command.project_id).parent / "status.json",
json.dumps(
{
"command_id": command.command_id,
"details": details,
"error_code": None,
"error_message": None,
"format_version": 1,
"project_id": command.project_id,
"snapshot_path": None,
"state": state,
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
+ "\n",
)
def _ready_for_apply(
controller: BatchController,
project_id: str,
inventory: DesignSnapshot,
) -> None:
close_command = controller.prepare_close(project_id)
_publish_bridge_snapshot(
controller,
close_command,
"project_closed",
inventory,
)
controller.confirm_close(project_id)
open_command = controller.prepare_open(project_id)
_publish_bridge_snapshot(
controller,
open_command,
"project_opened",
inventory,
)
controller.confirm_open(project_id)
def _ingest(controller: BatchController, tmp_path: Path, project_id: str, marker: Path) -> tuple[DesignSnapshot, Path]:
snapshot = _snapshot(project_id, marker, "OLD-1", f"{project_id}-inventory")
command = controller.prepare_inventory(project_id)
observed, path = _publish_bridge_snapshot(
controller,
command,
"inventory_exported",
snapshot,
)
controller.ingest_inventory(project_id)
return observed, path
def test_ingest_inventory_consumes_only_correlated_canonical_mailbox_evidence(
tmp_path: Path,
) -> None:
# Catches an arbitrary caller-selected snapshot authorizing inventory.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
command = controller.prepare_inventory(project_id)
observed, _ = _publish_bridge_snapshot(
controller,
command,
"inventory_exported",
_snapshot(project_id, markers[0], "OLD-1", "mailbox"),
)
synthetic = replace(observed, components=(_component("OTHER-1"),))
write_snapshot(tmp_path / "synthetic.json", synthetic)
controller.ingest_inventory(project_id)
assert controller._inventory(project_id) == observed
def test_ingest_inventory_rejects_status_from_another_command(tmp_path: Path) -> None:
# Catches schema-valid mailbox bytes being accepted without command correlation.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
command = controller.prepare_inventory(project_id)
_publish_bridge_snapshot(
controller,
replace(command, command_id="other-command"),
"inventory_exported",
_snapshot(project_id, markers[0], "OLD-1", "mailbox"),
)
with pytest.raises(ControllerError, match="mailbox_evidence_uncorrelated"):
controller.ingest_inventory(project_id)
def test_verify_apply_consumes_only_correlated_canonical_mailbox_evidence(
tmp_path: Path,
) -> None:
# Catches an arbitrary caller-selected post-dialog snapshot authorizing Apply.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
applied, _ = _publish_bridge_snapshot(
controller,
command,
"applied",
_snapshot(project_id, markers[0], "NEW-1", "mailbox", symbol="LIB:NEW"),
)
synthetic = replace(applied, components=(_component("WRONG-1", symbol="LIB:NEW"),))
write_snapshot(tmp_path / "synthetic-after.json", synthetic)
result = controller.verify_apply(project_id, 0)
assert result.passed is True
assert controller._last_verified_snapshot(project_id) == applied
def test_verify_apply_rejects_status_from_another_command(tmp_path: Path) -> None:
# Catches schema-valid post-dialog bytes being accepted without command correlation.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
_publish_bridge_snapshot(
controller,
replace(command, command_id="other-command"),
"applied",
_snapshot(project_id, markers[0], "NEW-1", "mailbox", symbol="LIB:NEW"),
)
with pytest.raises(ControllerError, match="mailbox_evidence_uncorrelated"):
controller.verify_apply(project_id, 0)
def test_prepare_inventory_publishes_a_protocol_valid_empty_command(tmp_path: Path) -> None:
# Catches inventory having no mailbox command producer before the first bridge pass.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
command = controller.prepare_inventory(project_id)
assert command.action == "inventory"
assert command.project_id == project_id
assert command.expected_count == 0
assert command.expected_logical_keys == ()
assert command.old_part_number == command.new_part_number == command.expected_fingerprint == ""
assert controller.command_path(project_id).is_file()
def test_prepare_inventory_preserves_unconsumed_mailbox_outputs(tmp_path: Path) -> None:
# Catches replay publication silently discarding bridge evidence not yet ingested.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
first = controller.prepare_inventory(project_id)
mailbox = controller.command_path(project_id).parent
status = mailbox / "status.json"
snapshot = mailbox / "snapshot.json"
status.write_bytes(b"{malformed-status")
snapshot.write_bytes(b"unconsumed-snapshot")
with pytest.raises(ControllerError, match="live_action_required"):
controller.prepare_inventory(project_id)
persisted = json.loads(controller.command_path(project_id).read_text(encoding="utf-8"))
assert persisted["command_id"] == first.command_id
assert status.read_bytes() == b"{malformed-status"
assert snapshot.read_bytes() == b"unconsumed-snapshot"
@pytest.mark.parametrize(
"boundary",
("before_command", "after_command", "after_prepared"),
)
def test_prepare_inventory_recovers_every_durable_publication_boundary(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
boundary: str,
) -> None:
# Catches an interrupted inventory publication replacing its durable command on retry.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
real_write_command = controller_module.write_command
real_append = controller.store._append
if boundary in {"before_command", "after_command"}:
def interrupt_command(path: Path, command: object) -> None:
if boundary == "after_command":
real_write_command(path, command)
raise RuntimeError(f"simulated inventory interruption {boundary}")
monkeypatch.setattr(controller_module, "write_command", interrupt_command)
else:
def interrupt_prepared(
recorded_project_id: str,
event: str,
payload: dict[str, object],
target: ProjectState,
descriptor: int,
) -> None:
real_append(recorded_project_id, event, payload, target, descriptor)
if event == "inventory_command_prepared":
raise RuntimeError("simulated inventory interruption after_prepared")
monkeypatch.setattr(controller.store, "_append", interrupt_prepared)
with pytest.raises(RuntimeError, match="simulated inventory interruption"):
controller.prepare_inventory(project_id)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
events = [
json.loads(line)
for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()
]
intent = next(
event
for event in events
if event["event"] == "inventory_command_publication_started"
)
intended_command_id = intent["payload"]["command"]["command_id"]
mailbox = controller.command_path(project_id).parent
if boundary == "after_command":
status_bytes = b"inventory-output-after-publication"
snapshot_bytes = b"inventory-snapshot-after-publication"
(mailbox / "status.json").write_bytes(status_bytes)
(mailbox / "snapshot.json").write_bytes(snapshot_bytes)
resumed = BatchController.open(controller.root)
recovered = resumed.prepare_inventory(project_id)
assert recovered.command_id == intended_command_id
assert json.loads(resumed.command_path(project_id).read_text(encoding="utf-8"))[
"command_id"
] == intended_command_id
final_events = [
json.loads(line)
for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()
]
assert sum(
event["event"] == "inventory_command_publication_started"
for event in final_events
) == 1
assert sum(
event["event"] == "inventory_command_prepared"
for event in final_events
) == 1
if boundary == "after_command":
assert (mailbox / "status.json").read_bytes() == status_bytes
assert (mailbox / "snapshot.json").read_bytes() == snapshot_bytes
assert not (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ intended_command_id
).exists()
def test_inventory_recovery_rejects_type_coerced_command_with_bridge_output(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches Python equality treating JSON booleans as protocol integers during recovery.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
real_write_command = controller_module.write_command
def interrupt_after_command(path: Path, command: object) -> None:
real_write_command(path, command)
raise RuntimeError("simulated interruption after inventory publication")
monkeypatch.setattr(controller_module, "write_command", interrupt_after_command)
with pytest.raises(RuntimeError, match="simulated interruption"):
controller.prepare_inventory(project_id)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
command_path = controller.command_path(project_id)
malformed = json.loads(command_path.read_text(encoding="utf-8"))
malformed["format_version"] = True
malformed["expected_count"] = False
command_path.write_text(
json.dumps(malformed, sort_keys=True, separators=(",", ":")) + "\n",
encoding="utf-8",
newline="",
)
status = command_path.parent / "status.json"
status_bytes = b"uncertain-bridge-output"
status.write_bytes(status_bytes)
resumed = BatchController.open(controller.root)
with pytest.raises(ControllerError, match="mailbox_reconcile_required"):
resumed.prepare_inventory(project_id)
assert resumed.project_state(project_id) is ProjectState.BLOCKED
assert resumed.action_for(project_id) == "reconcile"
persisted = json.loads(command_path.read_text(encoding="utf-8"))
assert persisted["format_version"] is True
assert persisted["expected_count"] is False
assert status.read_bytes() == status_bytes
def test_prepare_close_archives_consumed_inventory_outputs_before_command(
tmp_path: Path,
) -> None:
# Catches a replacement command being paired with stale inventory status/snapshot.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory_command = controller.prepare_inventory(project_id)
mailbox = controller.command_path(project_id).parent
status = mailbox / "status.json"
snapshot = mailbox / "snapshot.json"
inventory = _snapshot(project_id, markers[0], "OLD-1", "inventory")
inventory, _ = _publish_bridge_snapshot(
controller, inventory_command, "inventory_exported", inventory
)
status_bytes = status.read_bytes()
snapshot_bytes = snapshot.read_bytes()
controller.ingest_inventory(project_id)
_certify(certification_path, inventory)
close_command = controller.prepare_close(project_id)
history = (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ inventory_command.command_id
)
assert close_command.action == "close_project"
assert not status.exists()
assert not snapshot.exists()
assert (history / "status.json").read_bytes() == status_bytes
assert (history / "snapshot.json").read_bytes() == snapshot_bytes
assert json.loads(controller.command_path(project_id).read_text(encoding="utf-8"))[
"command_id"
] == close_command.command_id
def test_mailbox_retirement_recovers_after_one_file_was_archived(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a crash between status and snapshot moves stranding future publication.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory_command = controller.prepare_inventory(project_id)
mailbox = controller.command_path(project_id).parent
status = mailbox / "status.json"
snapshot = mailbox / "snapshot.json"
inventory = _snapshot(project_id, markers[0], "OLD-1", "inventory")
inventory, _ = _publish_bridge_snapshot(
controller, inventory_command, "inventory_exported", inventory
)
status_bytes = status.read_bytes()
snapshot_bytes = snapshot.read_bytes()
controller.ingest_inventory(project_id)
_certify(certification_path, inventory)
original_rename = Path.rename
interrupted = False
def interrupt_snapshot_move(source: Path, target: Path) -> Path:
nonlocal interrupted
if source == snapshot and not interrupted:
interrupted = True
raise OSError("simulated process loss")
return original_rename(source, target)
monkeypatch.setattr(Path, "rename", interrupt_snapshot_move)
with pytest.raises(ControllerError, match="mailbox_archive_failed"):
controller.prepare_close(project_id)
monkeypatch.setattr(Path, "rename", original_rename)
history = (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ inventory_command.command_id
)
assert not status.exists()
assert (history / "status.json").read_bytes() == status_bytes
assert snapshot.read_bytes() == snapshot_bytes
assert json.loads(controller.command_path(project_id).read_text(encoding="utf-8"))[
"command_id"
] == inventory_command.command_id
close_command = controller.prepare_close(project_id)
assert close_command.action == "close_project"
assert not snapshot.exists()
assert (history / "snapshot.json").read_bytes() == snapshot_bytes
events = [
json.loads(line)
for line in (controller.root / "events.jsonl")
.read_text(encoding="utf-8")
.splitlines()
]
assert sum(event["event"] == "mailbox_outputs_archived" for event in events) == 1
def test_mailbox_retirement_rejects_an_unsafe_recorded_command_id_before_moves(
tmp_path: Path,
) -> None:
# Catches corrupt durable command metadata moving mailbox evidence outside history.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory_command = controller.prepare_inventory(project_id)
mailbox = controller.command_path(project_id).parent
status = mailbox / "status.json"
snapshot = mailbox / "snapshot.json"
inventory = _snapshot(project_id, markers[0], "OLD-1", "inventory")
inventory, _ = _publish_bridge_snapshot(
controller, inventory_command, "inventory_exported", inventory
)
status_bytes = status.read_bytes()
snapshot_bytes = snapshot.read_bytes()
controller.ingest_inventory(project_id)
_certify(certification_path, inventory)
event_path = controller.root / "events.jsonl"
events = [
json.loads(line)
for line in event_path.read_text(encoding="utf-8").splitlines()
]
prepared = next(
event for event in events if event["event"] == "inventory_command_prepared"
)
prepared["payload"]["command_id"] = "..\\..\\..\\..\\escaped"
event_path.write_text(
"".join(
json.dumps(
event,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
+ "\n"
for event in events
),
encoding="utf-8",
newline="",
)
with pytest.raises(ControllerError, match="mailbox_evidence_uncorrelated"):
controller.prepare_close(project_id)
assert status.read_bytes() == status_bytes
assert snapshot.read_bytes() == snapshot_bytes
assert not (tmp_path / "escaped").exists()
def test_prepare_apply_retries_after_outputs_archived_before_command_publish(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a fully archived snapshot making the publication retry look stale.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory_command = controller.prepare_inventory(project_id)
mailbox = controller.command_path(project_id).parent
inventory = _snapshot(project_id, markers[0], "OLD-1", "inventory")
inventory, _ = _publish_bridge_snapshot(
controller, inventory_command, "inventory_exported", inventory
)
controller.ingest_inventory(project_id)
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
real_write_command = controller_module.write_command
def interrupt_publish(path: Path, command: object) -> None:
if getattr(command, "action", None) == "replace_part":
raise OSError("simulated process loss before command publication")
real_write_command(path, command) # pragma: no cover - defensive fixture path
monkeypatch.setattr(controller_module, "write_command", interrupt_publish)
with pytest.raises(OSError, match="simulated process loss"):
controller.prepare_apply(project_id, 0)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
history = (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ inventory_command.command_id
)
assert (history / "snapshot.json").exists()
assert not (mailbox / "snapshot.json").exists()
replacement = controller.prepare_apply(project_id, 0)
assert replacement.action == "replace_part"
assert json.loads(controller.command_path(project_id).read_text(encoding="utf-8"))[
"command_id"
] == replacement.command_id
@pytest.mark.parametrize(
"boundary",
("before_command", "after_command", "after_prepared"),
)
def test_prepare_apply_recovers_every_durable_publication_boundary(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
boundary: str,
) -> None:
# Catches a prepared replacement event and APPLYING projection committing separately.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
real_write_command = controller_module.write_command
real_append = controller.store._append
if boundary in {"before_command", "after_command"}:
def interrupt_command(path: Path, command: object) -> None:
if boundary == "after_command":
real_write_command(path, command)
raise RuntimeError(f"simulated apply interruption {boundary}")
monkeypatch.setattr(controller_module, "write_command", interrupt_command)
else:
def interrupt_prepared(
recorded_project_id: str,
event: str,
payload: dict[str, object],
target: ProjectState,
descriptor: int,
) -> None:
real_append(recorded_project_id, event, payload, target, descriptor)
if event == "command_prepared":
raise RuntimeError("simulated apply interruption after_prepared")
monkeypatch.setattr(controller.store, "_append", interrupt_prepared)
with pytest.raises(RuntimeError, match="simulated apply interruption"):
controller.prepare_apply(project_id, 0)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
events = [
json.loads(line)
for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()
]
intent = next(
event
for event in events
if event["event"] == "apply_command_publication_started"
)
intended_command_id = intent["payload"]["command"]["command_id"]
mailbox = controller.command_path(project_id).parent
resumed = BatchController.open(controller.root)
if boundary == "after_prepared":
assert resumed.project_state(project_id) is ProjectState.APPLYING
persisted = json.loads(resumed.command_path(project_id).read_text(encoding="utf-8"))
assert persisted["command_id"] == intended_command_id
else:
recovered = resumed.prepare_apply(project_id, 0)
assert recovered.command_id == intended_command_id
assert resumed.project_state(project_id) is ProjectState.APPLYING
after = _snapshot(
project_id,
markers[0],
"NEW-1",
"after-interrupted-publication",
symbol="LIB:NEW",
)
intended = command_from_data(intent["payload"]["command"])
_publish_bridge_snapshot(resumed, intended, "applied", after)
assert resumed.verify_apply(project_id, 0).passed is True
final_events = [
json.loads(line)
for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()
]
assert sum(
event["event"] == "apply_command_publication_started"
for event in final_events
) == 1
assert sum(event["event"] == "command_prepared" for event in final_events) == 1
assert not (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ intended_command_id
).exists()
def test_second_mapping_archives_verified_bridge_outputs_before_next_command(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a verified first mapping status blocking Continue for the second mapping.
mapping, projects, markers = _write_inputs(tmp_path)
mapping.write_text(
"old_part_number,new_part_number\nOLD-1,NEW-1\nOLD-2,NEW-2\n",
encoding="utf-8",
)
certification_path = tmp_path / "certifications.json"
controller = BatchController.create_job(
tmp_path / "run",
mapping,
projects,
certification_path=certification_path,
allowed_suffixes=(".prj",),
library_identity="lib-r42",
)
project_id = controller.projects[0].project_id
second_component = replace(
_component("OLD-2", symbol="LIB:OLD-2"),
logical_key="/MAIN|U2",
refdes="U2",
)
inventory = replace(
_snapshot(project_id, markers[0], "OLD-1", "inventory"),
components=(_component("OLD-1"), second_component),
)
mailbox = controller.command_path(project_id).parent
inventory_command = controller.prepare_inventory(project_id)
inventory, _ = _publish_bridge_snapshot(
controller, inventory_command, "inventory_exported", inventory
)
inventory_status = (mailbox / "status.json").read_bytes()
controller.ingest_inventory(project_id)
after_first = replace(
inventory,
fingerprint="after-first",
components=(_component("NEW-1", symbol="LIB:NEW"), second_component),
)
final = replace(
after_first,
fingerprint="after-second",
components=(
_component("NEW-1", symbol="LIB:NEW"),
replace(second_component, part_number="NEW-2", units=(
replace(second_component.units[0], symbol="LIB:NEW-2"),
)),
),
)
_record_certification(
certification_path,
inventory,
after_first,
MappingRule("OLD-1", "NEW-1"),
"/MAIN|U1",
)
_record_certification(
certification_path,
after_first,
final,
MappingRule("OLD-2", "NEW-2"),
"/MAIN|U2",
)
_ready_for_apply(controller, project_id, inventory)
first_command = controller.prepare_apply(project_id, 0)
assert not (mailbox / "status.json").exists()
assert not (mailbox / "snapshot.json").exists()
assert (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ inventory_command.command_id
/ "status.json"
).read_bytes() == inventory_status
after_first, _ = _publish_bridge_snapshot(
controller, first_command, "applied", after_first
)
first_status = (mailbox / "status.json").read_bytes()
assert controller.verify_apply(project_id, 0).passed
real_write_command = controller_module.write_command
interrupted_command_id: str | None = None
def interrupt_after_second_command(path: Path, command: object) -> None:
nonlocal interrupted_command_id
real_write_command(path, command)
interrupted_command_id = getattr(command, "command_id")
raise RuntimeError("simulated second mapping publication interruption")
monkeypatch.setattr(controller_module, "write_command", interrupt_after_second_command)
with pytest.raises(RuntimeError, match="second mapping publication interruption"):
controller.prepare_apply(project_id, 1)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
resumed = BatchController.open(controller.root)
second_command = resumed.prepare_apply(project_id, 1)
first_history = (
controller.root
/ "history"
/ "mailbox"
/ project_id
/ first_command.command_id
)
assert second_command.action == "replace_part"
assert second_command.command_id == interrupted_command_id
assert second_command.command_id != first_command.command_id
assert not (mailbox / "status.json").exists()
assert not (mailbox / "snapshot.json").exists()
assert (first_history / "status.json").read_bytes() == first_status
assert json.loads((first_history / "snapshot.json").read_text(encoding="utf-8"))[
"fingerprint"
] == after_first.fingerprint
def test_create_job_copies_accepted_inputs_and_records_their_hashes(tmp_path: Path) -> None:
# Catches jobs that depend on later edits to caller-owned mapping/project-list files.
mapping, projects, _ = _write_inputs(tmp_path)
expected = {
"mapping.csv": hashlib.sha256(mapping.read_bytes()).hexdigest(),
"projects.txt": hashlib.sha256(projects.read_bytes()).hexdigest(),
}
controller = BatchController.create_job(
tmp_path / "run",
mapping,
projects,
certification_path=tmp_path / "certifications.json",
allowed_suffixes=(".prj",),
)
mapping.write_text("old_part_number,new_part_number\nOTHER,CHANGED\n", encoding="utf-8")
reopened = BatchController.open(controller.root)
assert reopened.input_hashes == expected
assert reopened.rules == (MappingRule("OLD-1", "NEW-1"),)
assert reopened.project_state(reopened.projects[0].project_id) is ProjectState.INVENTORY_PENDING
@pytest.mark.parametrize("changed_input", ("mapping", "projects"))
def test_create_job_parses_and_copies_one_input_byte_snapshot(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
changed_input: str,
) -> None:
# Catches job.json being derived from different input bytes than immutable inputs/.
mapping, projects, markers = _write_inputs(tmp_path)
original_load_mapping = controller_module.load_mapping
original_load_projects = controller_module.load_projects
changed = False
def load_mapping_then_change(path: Path):
nonlocal changed
rules = original_load_mapping(path)
if changed_input == "mapping" and not changed:
mapping.write_text(
"old_part_number,new_part_number\nOTHER,CHANGED\n",
encoding="utf-8",
)
changed = True
return rules
def load_projects_then_change(path: Path, suffixes: tuple[str, ...]):
nonlocal changed
refs = original_load_projects(path, suffixes)
if changed_input == "projects" and not changed:
replacement_root = tmp_path / "projects" / "replacement"
replacement_root.mkdir()
replacement = replacement_root / "replacement.prj"
replacement.write_text("replacement\n", encoding="utf-8")
projects.write_text(f"{replacement.resolve()}\n", encoding="utf-8")
changed = True
return refs
monkeypatch.setattr(controller_module, "load_mapping", load_mapping_then_change)
monkeypatch.setattr(controller_module, "load_projects", load_projects_then_change)
controller = BatchController.create_job(
tmp_path / "run",
mapping,
projects,
certification_path=tmp_path / "certifications.json",
allowed_suffixes=(".prj",),
)
durable_job = json.loads((controller.root / "job.json").read_text(encoding="utf-8"))
copied_mapping = (controller.root / "inputs" / "mapping.csv").read_bytes()
copied_projects = (controller.root / "inputs" / "projects.txt").read_bytes()
assert durable_job["rules"] == [
{"old_part_number": rule.old_part_number, "new_part_number": rule.new_part_number}
for rule in controller.rules
]
assert durable_job["projects"] == [
{
"display_name": project.display_name,
"project_id": project.project_id,
"project_path": project.project_path,
}
for project in controller.projects
]
assert controller.input_hashes == {
"mapping.csv": hashlib.sha256(copied_mapping).hexdigest(),
"projects.txt": hashlib.sha256(copied_projects).hexdigest(),
}
assert parse_mapping_bytes(copied_mapping, mapping) == controller.rules
assert (
parse_projects_bytes(copied_projects, projects, (".prj",))
== controller.projects
)
assert controller.projects[0].project_path == str(markers[0].resolve())
def test_controller_reports_reject_event_log_corruption_introduced_after_open(
tmp_path: Path,
) -> None:
# Catches controller projections bypassing RunStore's canonical event validation.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
controller.store.append_event(project_id, "operator_note", {})
event_path = controller.root / "events.jsonl"
record = json.loads(event_path.read_text(encoding="utf-8"))
event_path.write_text(json.dumps(record, separators=(", ", ": ")) + "\n", encoding="utf-8")
with pytest.raises(StateError, match="corrupt_event_log"):
controller.report()
def test_inventory_identity_gate_and_all_project_barrier_are_durable(tmp_path: Path) -> None:
# Catches accepting a snapshot for the wrong project or enabling Apply before every inventory exists.
controller, markers, _ = _create_controller(tmp_path, project_count=2)
project_ids = tuple(project.project_id for project in controller.projects)
wrong = _snapshot("wrong", markers[0], "OLD-1", "wrong")
command = controller.prepare_inventory(project_ids[0])
_publish_bridge_snapshot(controller, command, "inventory_exported", wrong)
with pytest.raises(ControllerError, match="mailbox_evidence_uncorrelated"):
controller.ingest_inventory(project_ids[0])
(controller.command_path(project_ids[0]).parent / "status.json").unlink()
(controller.command_path(project_ids[0]).parent / "snapshot.json").unlink()
_ingest(controller, tmp_path, project_ids[0], markers[0])
assert controller.project_state(project_ids[0]) is ProjectState.INVENTORY_COMPLETE
_ingest(controller, tmp_path, project_ids[1], markers[1])
reopened = BatchController.open(controller.root)
assert [reopened.project_state(project_id) for project_id in project_ids] == [
ProjectState.APPLY_PENDING,
ProjectState.APPLY_PENDING,
]
impact = (controller.root / "reports" / "impact.csv").read_text(encoding="utf-8")
assert impact.count("OLD-1,NEW-1,1,1") == 2
def test_open_idempotently_finishes_an_interrupted_inventory_barrier(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a crash after the final durable inventory transition leaving every project stuck.
controller, markers, _ = _create_controller(tmp_path, project_count=2)
first, second = (project.project_id for project in controller.projects)
_ingest(controller, tmp_path, first, markers[0])
def interrupt_finalization() -> None:
raise RuntimeError("simulated process loss before impact publication")
monkeypatch.setattr(controller, "_finish_inventory_phase", interrupt_finalization)
with pytest.raises(RuntimeError, match="simulated process loss"):
_ingest(controller, tmp_path, second, markers[1])
assert RunStore(controller.root).project_state(first) is ProjectState.INVENTORY_COMPLETE
assert RunStore(controller.root).project_state(second) is ProjectState.INVENTORY_COMPLETE
resumed = BatchController.open(controller.root)
reopened_again = BatchController.open(controller.root)
assert resumed.project_state(first) is ProjectState.APPLY_PENDING
assert resumed.project_state(second) is ProjectState.APPLY_PENDING
assert reopened_again.project_state(first) is ProjectState.APPLY_PENDING
assert reopened_again.project_state(second) is ProjectState.APPLY_PENDING
assert (controller.root / "reports" / "impact.csv").exists()
def test_zero_match_project_completes_as_a_durable_verified_no_op(tmp_path: Path) -> None:
# Catches no-match projects remaining permanently apply_pending or receiving fake Apply/save events.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory = _snapshot(project_id, markers[0], "KEEP-1", "no-matches")
command = controller.prepare_inventory(project_id)
_publish_bridge_snapshot(controller, command, "inventory_exported", inventory)
controller.ingest_inventory(project_id)
assert controller.project_state(project_id) is ProjectState.VERIFIED_SAVED
assert controller.next_project() is None
reopened = BatchController.open(controller.root)
assert reopened.project_state(project_id) is ProjectState.VERIFIED_SAVED
records = [json.loads(line) for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()]
events = [record["event"] for record in records]
assert "no_changes_required" in events
assert "command_prepared" not in events
assert "saved_snapshot_verified" not in events
assert "external_checks_passed" not in events
def test_zero_match_interruption_before_completion_retries_on_open(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a crash between impact review and the no-change event stranding apply_pending forever.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory = _snapshot(project_id, markers[0], "KEEP-1", "no-match-interruption")
command = controller.prepare_inventory(project_id)
_publish_bridge_snapshot(controller, command, "inventory_exported", inventory)
def interrupt_completion(_: str) -> None:
raise RuntimeError("simulated process loss before no-change event")
monkeypatch.setattr(controller.store, "complete_no_changes", interrupt_completion)
with pytest.raises(RuntimeError, match="simulated process loss"):
controller.ingest_inventory(project_id)
assert RunStore(controller.root).project_state(project_id) is ProjectState.INVENTORY_COMPLETE
resumed = BatchController.open(controller.root)
assert resumed.project_state(project_id) is ProjectState.VERIFIED_SAVED
records = [json.loads(line) for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()]
assert [record["event"] for record in records].count("no_changes_required") == 1
assert not any(record["target_state"] == "apply_pending" for record in records)
def test_prepare_apply_blocks_uncertified_pair_then_stale_snapshot(tmp_path: Path) -> None:
# Catches Apply paths that proceed without an exact certification or after snapshot drift.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, source_path = _ingest(controller, tmp_path, project_id, markers[0])
_ready_for_apply(controller, project_id, inventory)
with pytest.raises(ControllerError, match="uncertified_mapping"):
controller.prepare_apply(project_id, 0)
_certify(certification_path, inventory)
write_snapshot(source_path, replace(inventory, fingerprint="stale"))
with pytest.raises(ControllerError, match="stale_project"):
controller.prepare_apply(project_id, 0)
def test_prepare_apply_writes_exact_atomic_command_after_verified_backup(tmp_path: Path) -> None:
# Catches commands that select a count instead of the exact inventoried logical identities.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
assert command.action == "replace_part"
assert command.expected_logical_keys == ("/MAIN|U1",)
assert command.expected_count == 1
assert command.expected_fingerprint == inventory.fingerprint
assert controller.project_state(project_id) is ProjectState.APPLYING
resumed = BatchController.open(controller.root)
assert resumed.action_for(project_id) == "reconcile"
next_project = resumed.next_project()
assert next_project is not None
assert next_project.project_id == project_id
persisted = json.loads(controller.command_path(project_id).read_text(encoding="utf-8"))
assert persisted["expected_logical_keys"] == ["/MAIN|U1"]
events = [json.loads(line) for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()]
assert any(event["event"] == "backup_verified" for event in events)
def test_last_verified_rule_recovers_to_save_after_transition_interruption(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
after = _snapshot(project_id, markers[0], "NEW-1", "after", symbol="LIB:NEW")
_publish_bridge_snapshot(controller, command, "applied", after)
original_transition = controller.store.transition
def interrupt_awaiting_save(candidate: str, target: ProjectState) -> None:
if target is ProjectState.AWAITING_SAVE_CONFIRMATION:
raise RuntimeError("simulated process loss")
original_transition(candidate, target)
monkeypatch.setattr(controller.store, "transition", interrupt_awaiting_save)
try:
controller.verify_apply(project_id, 0)
except RuntimeError:
pass
resumed = BatchController.open(controller.root)
step = resumed.operator_step()
assert step.command is not None
assert step.command.action == "save"
assert resumed.project_state(project_id) is ProjectState.AWAITING_SAVE_CONFIRMATION
def test_prepare_apply_replays_archive_move_completed_before_event(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
interrupted_command_id = controller._latest_published_command_event(project_id)["payload"]["command_id"]
original_append = controller.store.append_event
interrupted = False
def interrupt_archive(candidate: str, event: str, payload: dict[str, object]) -> None:
nonlocal interrupted
if event == "mailbox_outputs_archived" and not interrupted:
interrupted = True
raise RuntimeError("simulated process loss")
original_append(candidate, event, payload)
monkeypatch.setattr(controller.store, "append_event", interrupt_archive)
with pytest.raises(RuntimeError, match="simulated process loss"):
controller.prepare_apply(project_id, 0)
resumed = BatchController.open(controller.root)
command = resumed.prepare_apply(project_id, 0)
assert command.action == "replace_part"
archived = [
event
for event in resumed.store.events(project_id)
if event["event"] == "mailbox_outputs_archived"
and event["payload"].get("command_id") == interrupted_command_id
]
assert len(archived) == 1
def test_hard_verification_failure_blocks_save_and_requires_undo(tmp_path: Path) -> None:
# Catches a controller that advances after changed connectivity or merely trusts bridge success.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
unsafe = _snapshot(project_id, markers[0], "NEW-1", "after", symbol="LIB:NEW", net="CHANGED")
_publish_bridge_snapshot(controller, command, "applied", unsafe)
result = controller.verify_apply(project_id, 0)
assert result.passed is False
assert "connectivity_changed" in {issue.code for issue in result.issues}
assert controller.project_state(project_id) is ProjectState.BLOCKED
assert controller.action_for(project_id) == "prepare-undo"
assert "undo_required" in (controller.root / "events.jsonl").read_text(encoding="utf-8")
def test_exact_target_certification_is_rechecked_after_native_replace(tmp_path: Path) -> None:
# Catches certification checks that validate only the old/new Part Number strings.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory, target_symbol="LIB:NEW")
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
wrong_interface = _snapshot(project_id, markers[0], "NEW-1", "after", symbol="LIB:OTHER")
_publish_bridge_snapshot(controller, command, "applied", wrong_interface)
result = controller.verify_apply(project_id, 0)
assert result.passed is False
assert "uncertified_target" in {issue.code for issue in result.issues}
assert controller.project_state(project_id) is ProjectState.BLOCKED
def test_confirm_saved_rechecks_persisted_snapshot_and_external_checks(tmp_path: Path) -> None:
# Catches marking a project complete before the saved reread and project-level checks pass.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
after = _snapshot(project_id, markers[0], "NEW-1", "after", symbol="LIB:NEW")
after, _ = _publish_bridge_snapshot(controller, command, "applied", after)
assert controller.verify_apply(project_id, 0).passed is True
assert controller.project_state(project_id) is ProjectState.AWAITING_SAVE_CONFIRMATION
save_command = controller.prepare_save(project_id)
assert (save_command.action, save_command.command_id) == ("save", command.command_id)
_publish_bridge_snapshot(
controller,
save_command,
"saved",
after,
status_command=command,
)
state = controller.confirm_saved(project_id)
assert state is ProjectState.AWAITING_EXTERNAL_CHECKS
for check in ("packager", "erc", "netlist"):
check_command = controller.prepare_external_check(project_id, check)
_publish_bridge_snapshot(
controller,
check_command,
"external_check_passed",
after,
)
state = controller.confirm_external_check(project_id, check)
assert state is ProjectState.VERIFIED_SAVED
def test_action_for_finishes_external_checks_after_transition_interruption(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches all three durable check results leaving action_for with no remaining check to select.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
apply_command = controller.prepare_apply(project_id, 0)
after = _snapshot(
project_id,
markers[0],
"NEW-1",
"after",
symbol="LIB:NEW",
)
after, _ = _publish_bridge_snapshot(
controller,
apply_command,
"applied",
after,
)
assert controller.verify_apply(project_id, 0).passed is True
save_command = controller.prepare_save(project_id)
_publish_bridge_snapshot(
controller,
save_command,
"saved",
after,
status_command=apply_command,
)
assert controller.confirm_saved(project_id) is ProjectState.AWAITING_EXTERNAL_CHECKS
for check in ("packager", "erc"):
check_command = controller.prepare_external_check(project_id, check)
_publish_bridge_snapshot(
controller,
check_command,
"external_check_passed",
after,
)
assert (
controller.confirm_external_check(project_id, check)
is ProjectState.AWAITING_EXTERNAL_CHECKS
)
final_command = controller.prepare_external_check(project_id, "netlist")
_publish_bridge_snapshot(
controller,
final_command,
"external_check_passed",
after,
)
real_transition = controller.store.transition
def interrupt_final_transition(project: str, target: ProjectState) -> None:
if target is ProjectState.VERIFIED_SAVED:
raise RuntimeError("final transition interrupted")
real_transition(project, target)
monkeypatch.setattr(controller.store, "transition", interrupt_final_transition)
with pytest.raises(RuntimeError, match="final transition interrupted"):
controller.confirm_external_check(project_id, "netlist")
resumed = BatchController.open(controller.root)
assert resumed.project_state(project_id) is ProjectState.AWAITING_EXTERNAL_CHECKS
assert resumed.action_for(project_id) == "confirm-netlist"
assert (
resumed.confirm_external_check(project_id, "netlist")
is ProjectState.VERIFIED_SAVED
)
def test_confirm_saved_blocks_persisted_snapshot_drift(tmp_path: Path) -> None:
# Catches trusting a save acknowledgement without comparing the on-disk reread.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
after = _snapshot(project_id, markers[0], "NEW-1", "after", symbol="LIB:NEW")
after, _ = _publish_bridge_snapshot(controller, command, "applied", after)
controller.verify_apply(project_id, 0)
drifted = replace(after, components=(_component("NEW-1", symbol="LIB:NEW", net="DRIFT"),))
save_command = controller.prepare_save(project_id)
_publish_bridge_snapshot(
controller,
save_command,
"saved",
drifted,
status_command=command,
)
with pytest.raises(ControllerError, match="persisted_snapshot_mismatch"):
controller.confirm_saved(project_id)
assert controller.project_state(project_id) is ProjectState.BLOCKED
def test_resume_skips_verified_saved_and_reconciles_uncertain_real_store(tmp_path: Path) -> None:
# Catches resume logic that trusts an interrupted applying state or reprocesses completed projects.
controller, _, _ = _create_controller(tmp_path, project_count=2)
first, second = (project.project_id for project in controller.projects)
store = RunStore(controller.root)
for state in (
ProjectState.INVENTORY_COMPLETE,
ProjectState.APPLY_PENDING,
ProjectState.APPLYING,
ProjectState.AWAITING_SAVE_CONFIRMATION,
ProjectState.AWAITING_EXTERNAL_CHECKS,
ProjectState.VERIFIED_SAVED,
):
store.transition(first, state)
for state in (ProjectState.INVENTORY_COMPLETE, ProjectState.APPLY_PENDING, ProjectState.APPLYING):
store.transition(second, state)
resumed = BatchController.open(controller.root)
assert resumed.next_project().project_id == second
assert resumed.action_for(second) == "reconcile"
def test_next_project_does_not_let_one_blocked_project_stop_the_batch(tmp_path: Path) -> None:
# Catches queue selection that repeatedly returns a blocked project ahead of actionable work.
controller, _, _ = _create_controller(tmp_path, project_count=2)
first, second = (project.project_id for project in controller.projects)
store = RunStore(controller.root)
store.transition(first, ProjectState.BLOCKED)
store.transition(second, ProjectState.INVENTORY_COMPLETE)
store.transition(second, ProjectState.APPLY_PENDING)
resumed = BatchController.open(controller.root)
assert resumed.next_project().project_id == second
assert resumed.action_for(second) == "prepare-close"
def test_next_project_prioritizes_ingest_over_inventory_barrier_wait(tmp_path: Path) -> None:
# Catches an inventory_complete project masking the later project that can finish the barrier.
controller, _, _ = _create_controller(tmp_path, project_count=2)
first, second = (project.project_id for project in controller.projects)
RunStore(controller.root).transition(first, ProjectState.INVENTORY_COMPLETE)
resumed = BatchController.open(controller.root)
assert resumed.action_for(first) == "await_inventory"
assert resumed.next_project().project_id == second
assert resumed.action_for(second) == "ingest"
def test_prepare_apply_requires_correlated_close_backup_and_reopen(tmp_path: Path) -> None:
# Catches backup creation while the active project has no correlated closed evidence.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
with pytest.raises(ControllerError, match="backup_close_required"):
controller.prepare_apply(project_id, 0)
close_command = controller.prepare_close(project_id)
assert close_command.action == "close_project"
assert close_command.expected_fingerprint == inventory.fingerprint
_publish_bridge_snapshot(controller, close_command, "project_closed", inventory)
controller.confirm_close(project_id)
with pytest.raises(ControllerError, match="project_open_required"):
controller.prepare_apply(project_id, 0)
open_command = controller.prepare_open(project_id)
assert open_command.action == "open_project"
_publish_bridge_snapshot(controller, open_command, "project_opened", inventory)
controller.confirm_open(project_id)
reopened = BatchController.open(controller.root)
assert reopened.project_state(project_id) is ProjectState.APPLY_PENDING
records = [json.loads(line) for line in (controller.root / "events.jsonl").read_text(encoding="utf-8").splitlines()]
assert any(record["event"] == "project_closed" for record in records)
assert any(record["event"] == "backup_verified" for record in records)
assert any(record["event"] == "project_opened" for record in records)
def test_reconcile_requires_verified_undo_and_matching_fresh_inventory(tmp_path: Path) -> None:
# Catches uncertain modified state being accepted as a new zero-match baseline.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
unsafe, _ = _publish_bridge_snapshot(
controller,
command,
"applied",
_snapshot(
project_id,
markers[0],
"WRONG-1",
"unsafe",
symbol="LIB:NEW",
),
)
assert controller.verify_apply(project_id, 0).passed is False
with pytest.raises(ControllerError, match="recovery_evidence_missing"):
controller.reconcile(project_id)
undo_command = controller.prepare_undo(project_id)
assert (undo_command.action, undo_command.command_id) == ("undo", command.command_id)
assert undo_command.expected_fingerprint == unsafe.fingerprint
_publish_bridge_snapshot(
controller,
undo_command,
"undo_verified",
inventory,
status_command=command,
)
controller.confirm_undo(project_id)
controller.reconcile(project_id)
inventory_command = controller.prepare_inventory(project_id)
_publish_bridge_snapshot(
controller,
inventory_command,
"inventory_exported",
replace(inventory, components=()),
)
with pytest.raises(ControllerError, match="recovery_snapshot_mismatch"):
controller.ingest_inventory(project_id)
assert controller.project_state(project_id) is ProjectState.BLOCKED
def test_reconcile_treats_an_exposed_apply_intent_as_uncertain_mutation(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Catches a bridge-visible Apply command being treated as read-only when prepared-state commit was interrupted.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
real_write_command = controller_module.write_command
def interrupt_after_publish(path: Path, command: object) -> None:
real_write_command(path, command)
raise RuntimeError("Apply command was exposed")
monkeypatch.setattr(controller_module, "write_command", interrupt_after_publish)
with pytest.raises(RuntimeError, match="Apply command was exposed"):
controller.prepare_apply(project_id, 0)
monkeypatch.setattr(controller_module, "write_command", real_write_command)
mailbox = controller.command_path(project_id).parent
controller.command_path(project_id).write_bytes(b"{malformed-command")
(mailbox / "status.json").write_bytes(b"{malformed-status")
write_snapshot(
mailbox / "snapshot.json",
replace(
_snapshot(
project_id,
markers[0],
"WRONG-1",
"uncertain",
symbol="LIB:NEW",
),
fingerprint=snapshot_fingerprint(
_snapshot(
project_id,
markers[0],
"WRONG-1",
"uncertain",
symbol="LIB:NEW",
)
),
),
)
with pytest.raises(ControllerError, match="mailbox_reconcile_required"):
controller.prepare_apply(project_id, 0)
assert controller.action_for(project_id) == "prepare-close"
with pytest.raises(ControllerError, match="recovery_evidence_missing"):
controller.reconcile(project_id)
close_command = controller.prepare_close(project_id)
assert close_command.action == "close_without_save"
uncertain = _snapshot(
project_id,
markers[0],
"WRONG-1",
"uncertain",
symbol="LIB:NEW",
)
_publish_bridge_snapshot(
controller,
close_command,
"project_closed",
uncertain,
)
controller.confirm_close(project_id)
controller.reconcile(project_id)
assert controller.project_state(project_id) is ProjectState.INVENTORY_PENDING
assert controller.action_for(project_id) == "prepare-open"
def test_verified_undo_reconcile_starts_a_fresh_rule_cycle(tmp_path: Path) -> None:
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
_publish_bridge_snapshot(
controller,
command,
"applied",
_snapshot(project_id, markers[0], "WRONG-1", "unsafe", symbol="LIB:NEW"),
)
assert controller.verify_apply(project_id, 0).passed is False
undo_command = controller.prepare_undo(project_id)
_publish_bridge_snapshot(
controller,
undo_command,
"undo_verified",
inventory,
status_command=command,
)
controller.confirm_undo(project_id)
controller.reconcile(project_id)
fresh_command = controller.prepare_inventory(project_id)
_publish_bridge_snapshot(
controller,
fresh_command,
"inventory_exported",
inventory,
)
controller.ingest_inventory(project_id)
assert controller.project_state(project_id) is ProjectState.APPLY_PENDING
assert controller._verified_rule_indices(project_id) == ()
def test_operator_step_publishes_initial_inventory_idempotently(
tmp_path: Path,
) -> None:
# Catches an operator retry publishing a second command or appending duplicate events.
controller, _, _ = _create_controller(tmp_path)
project = controller.projects[0]
first = controller.operator_step()
event_count = len(controller.store.events())
repeated = BatchController.open(controller.root).operator_step()
assert first.complete is False
assert first.consumed_action is None
assert first.project == project
assert first.state is ProjectState.INVENTORY_PENDING
assert first.command is not None
assert first.command.action == "inventory"
assert repeated == first
assert len(controller.store.events()) == event_count
def test_operator_step_consumes_one_result_and_publishes_only_one_next_command(
tmp_path: Path,
) -> None:
# Catches one invocation draining multiple project mailboxes or publishing ahead.
controller, markers, _ = _create_controller(tmp_path, project_count=2)
first_project, second_project = controller.projects
first = controller.operator_step()
assert first.command is not None
_publish_bridge_snapshot(
controller,
first.command,
"inventory_exported",
_snapshot(
first_project.project_id,
markers[0],
"OLD-1",
"first-inventory",
),
)
before = controller.store.events()
advanced = controller.operator_step()
after = controller.store.events()
assert advanced.consumed_action == "inventory"
assert advanced.project == second_project
assert advanced.command is not None
assert advanced.command.action == "inventory"
appended = after[len(before) :]
assert sum(event["event"] == "inventory_observed" for event in appended) == 1
assert sum(
event["event"] == "inventory_command_prepared" for event in appended
) == 1
assert not any(
event["event"] == "inventory_observed"
and event["project_id"] == second_project.project_id
for event in appended
)
repeated = controller.operator_step()
assert repeated.command == advanced.command
assert repeated.consumed_action is None
assert len(controller.store.events()) == len(after)
def test_operator_step_drives_one_project_through_the_complete_live_protocol(
tmp_path: Path,
) -> None:
# Catches a missing public operator route between any two bridge-backed gates.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory_step = controller.operator_step()
assert inventory_step.command is not None
inventory, _ = _publish_bridge_snapshot(
controller,
inventory_step.command,
"inventory_exported",
_snapshot(project_id, markers[0], "OLD-1", "inventory"),
)
close_step = controller.operator_step()
assert close_step.consumed_action == "inventory"
assert close_step.command is not None
assert close_step.command.action == "close_project"
_certify(certification_path, inventory)
_publish_bridge_snapshot(
controller,
close_step.command,
"project_closed",
inventory,
)
open_step = controller.operator_step()
assert open_step.consumed_action == "close_project"
assert open_step.command is not None
assert open_step.command.action == "open_project"
_publish_bridge_snapshot(
controller,
open_step.command,
"project_opened",
inventory,
)
replace_step = controller.operator_step()
assert replace_step.consumed_action == "open_project"
assert replace_step.command is not None
assert replace_step.command.action == "replace_part"
applied, _ = _publish_bridge_snapshot(
controller,
replace_step.command,
"applied",
_snapshot(
project_id,
markers[0],
"NEW-1",
"applied",
symbol="LIB:NEW",
),
)
save_step = controller.operator_step()
assert save_step.consumed_action == "replace_part"
assert save_step.command is not None
assert save_step.command.action == "save"
repeated_save = controller.operator_step()
assert repeated_save.command == save_step.command
assert repeated_save.consumed_action is None
saved, _ = _publish_bridge_snapshot(
controller,
save_step.command,
"saved",
applied,
status_command=replace_step.command,
)
packager_step = controller.operator_step()
assert packager_step.consumed_action == "save"
assert packager_step.command is not None
assert packager_step.command.action == "packager"
current = packager_step
for action, next_action in (("packager", "erc"), ("erc", "netlist")):
assert current.command is not None
_publish_bridge_snapshot(
controller,
current.command,
"external_check_passed",
saved,
)
current = controller.operator_step()
assert current.consumed_action == action
assert current.command is not None
assert current.command.action == next_action
assert current.command is not None
_publish_bridge_snapshot(
controller,
current.command,
"external_check_passed",
saved,
)
complete = controller.operator_step()
assert complete.complete is True
assert complete.consumed_action == "netlist"
assert complete.project is None
assert complete.command is None
assert complete.state is None
assert controller.project_state(project_id) is ProjectState.VERIFIED_SAVED
repeated = controller.operator_step()
assert repeated.complete is True
assert repeated.consumed_action is None
assert repeated.project is None
assert repeated.command is None
def test_operator_step_consumes_verified_undo_then_publishes_fresh_inventory(
tmp_path: Path,
) -> None:
# Catches the operator loop stranding a blocked project after exact native Undo.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
apply_command = controller.prepare_apply(project_id, 0)
unsafe = _snapshot(
project_id,
markers[0],
"NEW-1",
"unsafe",
symbol="LIB:NEW",
net="CHANGED",
)
_publish_bridge_snapshot(controller, apply_command, "applied", unsafe)
assert controller.verify_apply(project_id, 0).passed is False
undo_step = controller.operator_step()
assert undo_step.command is not None
assert undo_step.command.action == "undo"
assert undo_step.consumed_action is None
_publish_bridge_snapshot(
controller,
undo_step.command,
"undo_verified",
inventory,
status_command=apply_command,
)
recovered = controller.operator_step()
assert recovered.consumed_action == "undo"
assert recovered.project == controller.projects[0]
assert recovered.command is not None
assert recovered.command.action == "inventory"
assert recovered.state is ProjectState.INVENTORY_PENDING
assert controller.action_for(project_id) == "ingest"
def test_operator_step_rejects_tampered_control_predecessor_status(
tmp_path: Path,
) -> None:
# Catches unrelated Apply evidence suppressing strict Save confirmation.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
apply_command = controller.prepare_apply(project_id, 0)
after = _snapshot(
project_id,
markers[0],
"NEW-1",
"after",
symbol="LIB:NEW",
)
_publish_bridge_snapshot(controller, apply_command, "applied", after)
assert controller.verify_apply(project_id, 0).passed is True
save_command = controller.prepare_save(project_id)
status_path = controller.command_path(project_id).parent / "status.json"
status = json.loads(status_path.read_text(encoding="utf-8"))
status["details"][2][1] = "another-job"
atomic_write_text(
status_path,
json.dumps(status, sort_keys=True, separators=(",", ":")) + "\n",
)
with pytest.raises(ControllerError) as captured:
controller.operator_step()
assert save_command.action == "save"
assert captured.value.code == "mailbox_evidence_uncorrelated"
assert controller.project_state(project_id) is ProjectState.AWAITING_SAVE_CONFIRMATION
@pytest.mark.parametrize(
("method_name", "arguments", "expected_code"),
(
("ingest_inventory", (), "mailbox_evidence_uncorrelated"),
("prepare_close", (), "invalid_state"),
("confirm_close", (), "close_command_missing"),
("prepare_open", (), "invalid_state"),
("confirm_open", (), "open_command_missing"),
("prepare_apply", (True,), "invalid_rule_index"),
("prepare_apply", (0,), "invalid_state"),
("verify_apply", (True,), "invalid_rule_index"),
("verify_apply", (0,), "invalid_state"),
("prepare_undo", (), "invalid_state"),
("confirm_undo", (), "invalid_state"),
("prepare_save", (), "invalid_state"),
("confirm_saved", (), "invalid_state"),
("prepare_external_check", ("packager",), "invalid_state"),
("confirm_external_check", ("packager",), "invalid_state"),
("restore", (), "invalid_state"),
("reconcile", (), "invalid_state"),
),
)
def test_public_operations_reject_calls_outside_their_durable_state(
tmp_path: Path,
method_name: str,
arguments: tuple[object, ...],
expected_code: str,
) -> None:
# Catches any public command bypassing its state or prerequisite gate.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
with pytest.raises(ControllerError) as captured:
getattr(controller, method_name)(project_id, *arguments)
assert captured.value.code == expected_code
def test_prepare_inventory_rejects_project_outside_inventory_pending(
tmp_path: Path,
) -> None:
# Catches a second inventory command bypassing the post-inventory state gate.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
_ingest(controller, tmp_path, project_id, markers[0])
assert controller.project_state(project_id) is ProjectState.APPLY_PENDING
with pytest.raises(ControllerError) as captured:
controller.prepare_inventory(project_id)
assert captured.value.code == "invalid_state"
def test_reconcile_rejects_mailbox_evidence_without_publication_intent(
tmp_path: Path,
) -> None:
# Catches read-only reconcile discarding mailbox bytes with no durable owner.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
controller.store.transition(project_id, ProjectState.BLOCKED)
status_path = controller.command_path(project_id).parent / "status.json"
status_path.write_bytes(b"unbound-mailbox-evidence")
event_count = len(controller.store.events())
with pytest.raises(ControllerError) as captured:
controller.reconcile(project_id)
assert captured.value.code == "mailbox_evidence_uncorrelated"
assert status_path.read_bytes() == b"unbound-mailbox-evidence"
assert controller.project_state(project_id) is ProjectState.BLOCKED
assert len(controller.store.events()) == event_count
@pytest.mark.parametrize(
"status_case",
("malformed", "terminal_without_snapshot", "incompatible_manual_route"),
)
def test_operator_step_does_not_treat_invalid_status_as_operator_wait(
tmp_path: Path,
status_case: str,
) -> None:
# Catches invalid bridge output being preserved as an authorized operator wait.
controller, _, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
command = controller.prepare_inventory(project_id)
status_path = controller.command_path(project_id).parent / "status.json"
if status_case == "malformed":
status_path.write_bytes(b"{malformed-status")
elif status_case == "terminal_without_snapshot":
_publish_waiting_status(controller, command, "inventory_exported")
else:
_publish_waiting_status(
controller,
command,
"manual_action_required",
manual_route="manual_native_dialog",
)
event_count = len(controller.store.events())
with pytest.raises(ControllerError) as captured:
controller.operator_step()
assert captured.value.code == "mailbox_evidence_uncorrelated"
assert controller.project_state(project_id) is ProjectState.INVENTORY_PENDING
assert len(controller.store.events()) == event_count
@pytest.mark.parametrize(
("bridge_state", "manual_route"),
(
("native_dialog_opening", None),
("manual_action_required", "manual_exact_selection"),
("manual_action_required", "manual_native_dialog"),
),
)
def test_operator_step_preserves_correlated_replace_part_waiting_status(
tmp_path: Path,
bridge_state: str,
manual_route: str | None,
) -> None:
# Catches an explicit operator wait being consumed as completed bridge evidence.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
command = controller.prepare_apply(project_id, 0)
_publish_waiting_status(
controller,
command,
bridge_state,
manual_route=manual_route,
)
event_count = len(controller.store.events())
waiting = BatchController.open(controller.root).operator_step()
assert waiting.complete is False
assert waiting.command == command
assert waiting.consumed_action is None
assert waiting.bridge_state == bridge_state
assert waiting.manual_route == manual_route
assert waiting.instruction == "complete the current native action"
assert len(controller.store.events()) == event_count
def test_confirm_close_rejects_snapshot_drift_before_backup(tmp_path: Path) -> None:
# Catches a close result authorizing backup after the live project changed.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
command = controller.prepare_close(project_id)
drifted = replace(inventory, components=(_component("OLD-1", net="DRIFT"),))
_publish_bridge_snapshot(controller, command, "project_closed", drifted)
with pytest.raises(ControllerError) as captured:
controller.confirm_close(project_id)
assert captured.value.code == "close_snapshot_mismatch"
assert controller.project_state(project_id) is ProjectState.APPLY_PENDING
assert not any(
event["event"] in {"project_closed", "backup_verified"}
for event in controller.store.events(project_id)
)
def test_confirm_open_blocks_snapshot_drift_after_verified_backup(tmp_path: Path) -> None:
# Catches Apply starting from a reopened project that differs from its trusted snapshot.
controller, markers, _ = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
close_command = controller.prepare_close(project_id)
_publish_bridge_snapshot(controller, close_command, "project_closed", inventory)
controller.confirm_close(project_id)
open_command = controller.prepare_open(project_id)
drifted = replace(inventory, components=(_component("OLD-1", net="DRIFT"),))
_publish_bridge_snapshot(controller, open_command, "project_opened", drifted)
with pytest.raises(ControllerError) as captured:
controller.confirm_open(project_id)
assert captured.value.code == "project_open_mismatch"
assert controller.project_state(project_id) is ProjectState.BLOCKED
assert controller.store.events(project_id)[-1]["event"] == "project_open_mismatch"
def test_confirm_undo_rejects_inexact_restoration(tmp_path: Path) -> None:
# Catches a successful Undo status legitimizing a project that was not exactly restored.
controller, markers, certification_path = _create_controller(tmp_path)
project_id = controller.projects[0].project_id
inventory, _ = _ingest(controller, tmp_path, project_id, markers[0])
_certify(certification_path, inventory)
_ready_for_apply(controller, project_id, inventory)
apply_command = controller.prepare_apply(project_id, 0)
unsafe, _ = _publish_bridge_snapshot(
controller,
apply_command,
"applied",
_snapshot(
project_id,
markers[0],
"WRONG-1",
"unsafe",
symbol="LIB:NEW",
),
)
assert controller.verify_apply(project_id, 0).passed is False
undo_command = controller.prepare_undo(project_id)
still_drifted = replace(unsafe, components=(_component("OLD-1", net="DRIFT"),))
_publish_bridge_snapshot(
controller,
undo_command,
"undo_verified",
still_drifted,
status_command=apply_command,
)
with pytest.raises(ControllerError) as captured:
controller.confirm_undo(project_id)
assert captured.value.code == "undo_verification_failed"
assert controller.project_state(project_id) is ProjectState.BLOCKED
assert controller.store.events(project_id)[-1]["event"] == "undo_verification_failed"