Files
dxdesigner-part-replacement/tests/dsh/test_pr27_run_finalization.py
T
阿白叔 f9d5e56bb8
Core regression / Python core, intervention UI and release configuration (push) Failing after 4m4s
Core regression / JavaScript syntax (push) Successful in 5s
Core regression / Python core, intervention UI and release configuration (pull_request) Failing after 4m20s
Core regression / JavaScript syntax (pull_request) Successful in 4s
fix: address PR27 lifecycle, BOM and netlist review findings
2026-11-13 16:03:59 +08:00

308 lines
16 KiB
Python

"""Ordinary I/O failures close only their bound execution attempt."""
import json
import sqlite3
from threading import Event
import pytest
from dx_productivity.dsh.execution_runs import execute_confirmed_run, execute_incident_retry
from dx_productivity.dsh.run_journal import RunJournal
from dx_productivity.dsh.scheduling import ScheduleStore
from dx_productivity.dsh.task_lifecycle import preflight
from dx_productivity.trial_task_center import task_queue
from dx_productivity.trial_task_center.errors import TaskCenterError
from dx_productivity.trial_task_center.task_queue import TaskQueueResult, TaskQueueEntry
from test_run_journal import project_service, checkpoint
from test_incident_rule_retry import rule_run
def fail(message):
raise OSError(message)
def test_resume_checkpoint_repair_failure_closes_durable_run(project_service, monkeypatch):
service, _, data, ids, hashes = project_service
run_id = 'a' * 32
journal = RunJournal(data)
journal.begin(run_id, ids, hashes)
checkpoint(data, run_id, ids, hashes)
prior = {'status': 'paused', 'tasks': [], 'entries': [
{'task_id': key, 'state': 'pending', 'message': ''} for key in ids]}
journal.finish(run_id, prior)
resume = journal.inspect(service, run_id)
monkeypatch.setattr(RunJournal, 'repair_retry_checkpoint', lambda *args: fail('repair failed'))
with pytest.raises(OSError, match='repair failed'):
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
row = RunJournal(data).get(run_id)
assert row['status'] == 'interrupted'
assert row['result']['entries'] == prior['entries']
assert row['result']['tasks'] == prior['tasks']
@pytest.mark.parametrize('rebound', [False, True])
@pytest.mark.parametrize('boundary', ['inspect', 'bound'])
def test_scheduled_resume_inspection_failure_closes_only_bound_claim(project_service, monkeypatch, rebound, boundary):
service, _, data, ids, hashes = project_service
run_id = 'b' * 32
schedules = ScheduleStore(data)
schedule = schedules.save({'task_ids': ids, 'priority': 'normal',
'not_before': '2026-01-01T00:00:00+00:00'}, hashes)
schedules.claim(schedule['schedule_id'], run_id=run_id)
journal = RunJournal(data)
journal.begin(run_id, ids, hashes, schedule_id=schedule['schedule_id'])
checkpoint(data, run_id, ids, hashes)
journal.finish(run_id, {'status': 'paused', 'entries': []})
resume = journal.inspect(service, run_id)
if rebound:
with schedules._connect() as db:
db.execute('UPDATE schedules SET run_id=? WHERE schedule_id=?', ('c' * 32, schedule['schedule_id']))
if boundary == 'inspect':
monkeypatch.setattr(RunJournal, 'inspect', lambda *args: fail('inspection failed'))
else:
from dx_productivity.dsh import execution_runs
bound, calls = execution_runs._bound_schedule, []
def unavailable_once(*args):
calls.append(True)
if len(calls) == 1:
fail('binding read failed')
return bound(*args)
monkeypatch.setattr(execution_runs, '_bound_schedule', unavailable_once)
with pytest.raises(TaskCenterError if rebound else OSError):
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
row = schedules.get(schedule['schedule_id'])
assert row['status'] == ('claimed' if rebound else 'interrupted')
assert row['run_id'] == ('c' * 32 if rebound else run_id)
@pytest.mark.parametrize('boundary', ['checkpoint_before', 'execute', 'checkpoint_after', 'finish'])
@pytest.mark.parametrize('scheduled', [False, True])
def test_retry_io_failure_preserves_bound_evidence_and_never_replays(rule_run, monkeypatch, boundary, scheduled):
case = rule_run(scheduled=scheduled)
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
selected = service.reserve_problem_retry(task_id, source='dsh')
service.prepare_failed_quick_run(task_id, operation_keys=selected)
prepared = service.task_context(task_id)
p = prepared.progress
expected_item = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number,
'part_label' if prepared.task.copy_part_number_to_value else 'preserve')
journal = RunJournal(root)
old = journal.get(run_id)['result']
save, finish = task_queue._save, RunJournal.finish
if boundary == 'execute':
monkeypatch.setattr(service, 'execute_quick_run_queue', lambda *args, **kwargs: fail('retry boundary'))
elif boundary.startswith('checkpoint'):
def broken_save(path, state):
expected = 'running' if boundary == 'checkpoint_before' else 'completed'
if state['entries'][task_id]['state'] == expected:
fail('retry boundary')
return save(path, state)
monkeypatch.setattr(task_queue, '_save', broken_save)
else:
def broken_finish(self, key, receipt):
if receipt.get('execution_status') == 'completed':
fail('retry boundary')
return finish(self, key, receipt)
monkeypatch.setattr(RunJournal, 'finish', broken_finish)
with pytest.raises(OSError, match='retry boundary'):
execute_incident_retry(service, root, task_id, run_id, expected_item=expected_item,
operation_keys=selected, stop_event=Event())
row = RunJournal(root).get(run_id)
assert row['status'] == 'interrupted'
assert row['task_ids'] == [task_id] and row['run_id'] == run_id
assert row['result']['retry']['operation_keys'] == sorted(selected)
assert row['result']['netlist_verification'] == old['netlist_verification']
assert row['result']['tasks'][0]['saved_items'] >= old['tasks'][0]['saved_items']
expected_calls = ['OLD-OK', 'OLD-BAD'] + (['OLD-BAD'] if boundary in {'checkpoint_after', 'finish'} else [])
assert case['calls'] == expected_calls
if boundary in {'checkpoint_after', 'finish'}:
assert row['result']['tasks'][0]['saved_items'] == 2
if scheduled:
schedule = ScheduleStore(root).get(case['schedule']['schedule_id'])
assert schedule['status'] == 'interrupted' and schedule['run_id'] == run_id
assert service.reserve_problem_retry(task_id, source='dsh') == ()
assert case['calls'] == expected_calls
if boundary in {'checkpoint_after', 'finish'}:
monkeypatch.setattr(task_queue, '_save', save)
monkeypatch.setattr(RunJournal, 'finish', finish)
resumed = execute_confirmed_run(service, root, [task_id],
preflight(service, [task_id], allow_completed=True), Event(), run_id,
resume=RunJournal(root).inspect(service, run_id))
assert resumed['status'] == 'completed'
assert case['calls'] == expected_calls, 'A saved retry may finalize but never dispatch again'
@pytest.mark.parametrize('failed_write', [1, 2])
@pytest.mark.parametrize('scheduled', [False, True])
def test_final_receipt_write_failure_is_durable_and_propagates(project_service, monkeypatch, failed_write, scheduled):
service, _, data, ids, hashes = project_service
run_id = 'd' * 32
schedule = ScheduleStore(data).save({'task_ids': ids, 'priority': 'normal',
'not_before': '2026-01-01T00:00:00+00:00'}, hashes) if scheduled else None
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *a, **kw:
TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False))
finish = RunJournal.finish
calls = 0
def broken_finish(self, key, receipt):
nonlocal calls
calls += 1
if calls == failed_write:
fail('receipt boundary')
return finish(self, key, receipt)
monkeypatch.setattr(RunJournal, 'finish', broken_finish)
with pytest.raises(OSError, match='receipt boundary'):
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, schedule=schedule)
row = RunJournal(data).get(run_id)
assert row['status'] == 'interrupted'
assert [entry['state'] for entry in row['result']['entries']] == ['completed'] * len(ids)
assert row['result']['code'] == 'receipt_write_error'
if scheduled:
after = ScheduleStore(data).get(schedule['schedule_id'])
assert after['status'] == 'interrupted' and after['run_id'] == run_id
@pytest.mark.parametrize('damage', ['scope', 'unknown'])
def test_saved_retry_with_changed_scope_or_unknown_proof_records_observation_without_replay(rule_run, monkeypatch, damage):
case = rule_run(scheduled=True)
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
selected = service.reserve_problem_retry(task_id, source='dsh')
service.prepare_failed_quick_run(task_id, operation_keys=selected)
prepared = service.task_context(task_id)
p = prepared.progress
expected = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number,
'part_label' if prepared.task.copy_part_number_to_value else 'preserve')
execute = service.execute_quick_run_queue
path = root / 'task-queues' / (run_id + '.json')
def damage_after_save(*args, **kwargs):
outcome = execute(*args, **kwargs)
assert service.task_context(task_id).progress.saved_items == 2
if damage == 'scope':
state = json.loads(path.read_text(encoding='utf-8'))
state['task_ids'] = ['f' * 32]
path.write_text(json.dumps(state), encoding='utf-8')
else:
case['store']._connection.execute('UPDATE execution_operations SET state=? WHERE operation_key=?',
('uncertain', selected[0]))
fail('saved outcome now uncertain')
return outcome
monkeypatch.setattr(service, 'execute_quick_run_queue', damage_after_save)
with pytest.raises((TaskCenterError, OSError)):
execute_incident_retry(service, root, task_id, run_id, expected_item=expected,
operation_keys=selected, stop_event=Event())
journal = RunJournal(root)
row = journal.get(run_id)
assert row['status'] == 'interrupted'
assert row['result']['tasks'][0]['saved_items'] == 1, 'Unsafe evidence cannot replace the authorized baseline'
if damage == 'scope':
assert row['result']['retry_observation']['saved_items'] == 2
assert row['result']['retry_observation']['task_id'] == task_id
inspected = journal.inspect(service, run_id)
assert not inspected['resume_ready']
with pytest.raises(TaskCenterError):
execute_confirmed_run(service, root, [task_id], preflight(service, [task_id], allow_completed=True),
Event(), run_id, resume=inspected)
assert case['calls'] == ['OLD-OK', 'OLD-BAD', 'OLD-BAD']
schedule = ScheduleStore(root).get(case['schedule']['schedule_id'])
assert schedule['status'] == 'interrupted' and schedule['run_id'] == run_id
def test_scheduled_resume_terminal_publication_failure_closes_claim(project_service, monkeypatch):
service, _, data, ids, hashes = project_service
run_id = 'e' * 32
schedules = ScheduleStore(data)
schedule = schedules.save({'task_ids': ids, 'priority': 'normal',
'not_before': '2026-01-01T00:00:00+00:00'}, hashes)
schedules.claim(schedule['schedule_id'], run_id=run_id)
journal = RunJournal(data)
journal.begin(run_id, ids, hashes, schedule_id=schedule['schedule_id'])
checkpoint(data, run_id, ids, hashes)
journal.finish(run_id, {'status': 'paused', 'entries': []})
resume = journal.inspect(service, run_id)
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *a, **kw:
TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False))
finish, calls = ScheduleStore.finish, []
def broken_once(self, *args, **kwargs):
calls.append(True)
if len(calls) == 1:
fail('schedule publication failed')
return finish(self, *args, **kwargs)
monkeypatch.setattr(ScheduleStore, 'finish', broken_once)
with pytest.raises(OSError, match='schedule publication failed'):
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
after = schedules.get(schedule['schedule_id'])
assert after['status'] == 'interrupted' and after['run_id'] == run_id
assert journal.get(run_id)['status'] == 'completed', 'Known executor receipt stays durable'
@pytest.mark.parametrize('operation', ['begin', 'resume', 'finish'])
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
def test_running_producer_returns_committed_record_without_fallible_tail_read(project_service, monkeypatch, operation, error_type):
_, _, data, ids, hashes = project_service
run_id = '1' * 32
journal = RunJournal(data)
if operation in {'resume', 'finish'}:
journal.begin(run_id, ids, hashes)
if operation == 'resume':
journal.finish(run_id, {'status': 'paused', 'entries': []})
read, failures = RunJournal.get, []
def unavailable_once(self, key):
row = read(self, key)
if row and row['status'] == ('completed' if operation == 'finish' else 'running') and not failures:
failures.append(True)
raise error_type('post-commit read unavailable')
return row
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
returned = (journal.begin(run_id, ids, hashes) if operation == 'begin' else journal.resume(run_id)
if operation == 'resume' else journal.finish(run_id, {'status': 'completed', 'entries': []}))
assert returned == read(RunJournal(data), run_id)
assert returned['status'] == ('completed' if operation == 'finish' else 'running')
assert not failures, 'A committed mutation must not need a second connection/read to acknowledge it'
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
def test_retry_producer_returns_real_intent_without_post_commit_read(rule_run, monkeypatch, error_type):
case = rule_run()
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
selected = service.reserve_problem_retry(task_id, source='dsh')
service.prepare_failed_quick_run(task_id, operation_keys=selected)
read, failures = RunJournal.get, []
def unavailable_once(self, key):
row = read(self, key)
if row and row['status'] == 'running' and not failures:
failures.append(True)
raise error_type('post-commit retry read unavailable')
return row
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
returned = RunJournal(root).begin_retry(service, run_id, task_id, selected)
assert returned == read(RunJournal(root), run_id)
assert returned['status'] == 'running' and returned['result']['retry']['operation_keys'] == sorted(selected)
assert case['calls'] == ['OLD-OK', 'OLD-BAD']
assert not failures
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
def test_post_execution_read_failure_cannot_lose_known_outcome(project_service, monkeypatch, error_type):
service, _, data, ids, _ = project_service
run_id = '2' * 32
report = {ids[0]: {'status': 'verified', 'verified': True, 'report_path': 'known-save.json'}}
read, failures, executed = RunJournal.get, [], []
def executor(*args, **kwargs):
executed.append(True)
return TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False,
netlist_verification=report)
def unavailable_once(self, key):
if executed and not failures:
failures.append(True)
raise error_type('post-execution read unavailable')
return read(self, key)
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', executor)
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
try:
receipt = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id)
except (OSError, sqlite3.OperationalError):
receipt = read(RunJournal(data), run_id)['result']
row = read(RunJournal(data), run_id)
assert row['status'] in {'completed', 'interrupted'}
assert [entry['state'] for entry in row['result']['entries']] == ['completed'] * len(ids)
assert row['result']['netlist_verification'] == report
assert len(executed) == 1