Core regression / Python core, intervention UI and release configuration (push) Failing after 4m4s
Core regression / JavaScript syntax (push) Successful in 5s
Core regression / Python core, intervention UI and release configuration (pull_request) Failing after 4m20s
Core regression / JavaScript syntax (pull_request) Successful in 4s
308 lines
16 KiB
Python
308 lines
16 KiB
Python
"""Ordinary I/O failures close only their bound execution attempt."""
|
|
import json
|
|
import sqlite3
|
|
from threading import Event
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.dsh.execution_runs import execute_confirmed_run, execute_incident_retry
|
|
from dx_productivity.dsh.run_journal import RunJournal
|
|
from dx_productivity.dsh.scheduling import ScheduleStore
|
|
from dx_productivity.dsh.task_lifecycle import preflight
|
|
from dx_productivity.trial_task_center import task_queue
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
from dx_productivity.trial_task_center.task_queue import TaskQueueResult, TaskQueueEntry
|
|
from test_run_journal import project_service, checkpoint
|
|
from test_incident_rule_retry import rule_run
|
|
|
|
|
|
def fail(message):
|
|
raise OSError(message)
|
|
|
|
|
|
def test_resume_checkpoint_repair_failure_closes_durable_run(project_service, monkeypatch):
|
|
service, _, data, ids, hashes = project_service
|
|
run_id = 'a' * 32
|
|
journal = RunJournal(data)
|
|
journal.begin(run_id, ids, hashes)
|
|
checkpoint(data, run_id, ids, hashes)
|
|
prior = {'status': 'paused', 'tasks': [], 'entries': [
|
|
{'task_id': key, 'state': 'pending', 'message': ''} for key in ids]}
|
|
journal.finish(run_id, prior)
|
|
resume = journal.inspect(service, run_id)
|
|
monkeypatch.setattr(RunJournal, 'repair_retry_checkpoint', lambda *args: fail('repair failed'))
|
|
with pytest.raises(OSError, match='repair failed'):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
|
|
row = RunJournal(data).get(run_id)
|
|
assert row['status'] == 'interrupted'
|
|
assert row['result']['entries'] == prior['entries']
|
|
assert row['result']['tasks'] == prior['tasks']
|
|
|
|
|
|
@pytest.mark.parametrize('rebound', [False, True])
|
|
@pytest.mark.parametrize('boundary', ['inspect', 'bound'])
|
|
def test_scheduled_resume_inspection_failure_closes_only_bound_claim(project_service, monkeypatch, rebound, boundary):
|
|
service, _, data, ids, hashes = project_service
|
|
run_id = 'b' * 32
|
|
schedules = ScheduleStore(data)
|
|
schedule = schedules.save({'task_ids': ids, 'priority': 'normal',
|
|
'not_before': '2026-01-01T00:00:00+00:00'}, hashes)
|
|
schedules.claim(schedule['schedule_id'], run_id=run_id)
|
|
journal = RunJournal(data)
|
|
journal.begin(run_id, ids, hashes, schedule_id=schedule['schedule_id'])
|
|
checkpoint(data, run_id, ids, hashes)
|
|
journal.finish(run_id, {'status': 'paused', 'entries': []})
|
|
resume = journal.inspect(service, run_id)
|
|
if rebound:
|
|
with schedules._connect() as db:
|
|
db.execute('UPDATE schedules SET run_id=? WHERE schedule_id=?', ('c' * 32, schedule['schedule_id']))
|
|
if boundary == 'inspect':
|
|
monkeypatch.setattr(RunJournal, 'inspect', lambda *args: fail('inspection failed'))
|
|
else:
|
|
from dx_productivity.dsh import execution_runs
|
|
bound, calls = execution_runs._bound_schedule, []
|
|
def unavailable_once(*args):
|
|
calls.append(True)
|
|
if len(calls) == 1:
|
|
fail('binding read failed')
|
|
return bound(*args)
|
|
monkeypatch.setattr(execution_runs, '_bound_schedule', unavailable_once)
|
|
with pytest.raises(TaskCenterError if rebound else OSError):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
|
|
row = schedules.get(schedule['schedule_id'])
|
|
assert row['status'] == ('claimed' if rebound else 'interrupted')
|
|
assert row['run_id'] == ('c' * 32 if rebound else run_id)
|
|
|
|
|
|
@pytest.mark.parametrize('boundary', ['checkpoint_before', 'execute', 'checkpoint_after', 'finish'])
|
|
@pytest.mark.parametrize('scheduled', [False, True])
|
|
def test_retry_io_failure_preserves_bound_evidence_and_never_replays(rule_run, monkeypatch, boundary, scheduled):
|
|
case = rule_run(scheduled=scheduled)
|
|
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
|
|
selected = service.reserve_problem_retry(task_id, source='dsh')
|
|
service.prepare_failed_quick_run(task_id, operation_keys=selected)
|
|
prepared = service.task_context(task_id)
|
|
p = prepared.progress
|
|
expected_item = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number,
|
|
'part_label' if prepared.task.copy_part_number_to_value else 'preserve')
|
|
journal = RunJournal(root)
|
|
old = journal.get(run_id)['result']
|
|
save, finish = task_queue._save, RunJournal.finish
|
|
if boundary == 'execute':
|
|
monkeypatch.setattr(service, 'execute_quick_run_queue', lambda *args, **kwargs: fail('retry boundary'))
|
|
elif boundary.startswith('checkpoint'):
|
|
def broken_save(path, state):
|
|
expected = 'running' if boundary == 'checkpoint_before' else 'completed'
|
|
if state['entries'][task_id]['state'] == expected:
|
|
fail('retry boundary')
|
|
return save(path, state)
|
|
monkeypatch.setattr(task_queue, '_save', broken_save)
|
|
else:
|
|
def broken_finish(self, key, receipt):
|
|
if receipt.get('execution_status') == 'completed':
|
|
fail('retry boundary')
|
|
return finish(self, key, receipt)
|
|
monkeypatch.setattr(RunJournal, 'finish', broken_finish)
|
|
with pytest.raises(OSError, match='retry boundary'):
|
|
execute_incident_retry(service, root, task_id, run_id, expected_item=expected_item,
|
|
operation_keys=selected, stop_event=Event())
|
|
row = RunJournal(root).get(run_id)
|
|
assert row['status'] == 'interrupted'
|
|
assert row['task_ids'] == [task_id] and row['run_id'] == run_id
|
|
assert row['result']['retry']['operation_keys'] == sorted(selected)
|
|
assert row['result']['netlist_verification'] == old['netlist_verification']
|
|
assert row['result']['tasks'][0]['saved_items'] >= old['tasks'][0]['saved_items']
|
|
expected_calls = ['OLD-OK', 'OLD-BAD'] + (['OLD-BAD'] if boundary in {'checkpoint_after', 'finish'} else [])
|
|
assert case['calls'] == expected_calls
|
|
if boundary in {'checkpoint_after', 'finish'}:
|
|
assert row['result']['tasks'][0]['saved_items'] == 2
|
|
if scheduled:
|
|
schedule = ScheduleStore(root).get(case['schedule']['schedule_id'])
|
|
assert schedule['status'] == 'interrupted' and schedule['run_id'] == run_id
|
|
assert service.reserve_problem_retry(task_id, source='dsh') == ()
|
|
assert case['calls'] == expected_calls
|
|
if boundary in {'checkpoint_after', 'finish'}:
|
|
monkeypatch.setattr(task_queue, '_save', save)
|
|
monkeypatch.setattr(RunJournal, 'finish', finish)
|
|
resumed = execute_confirmed_run(service, root, [task_id],
|
|
preflight(service, [task_id], allow_completed=True), Event(), run_id,
|
|
resume=RunJournal(root).inspect(service, run_id))
|
|
assert resumed['status'] == 'completed'
|
|
assert case['calls'] == expected_calls, 'A saved retry may finalize but never dispatch again'
|
|
|
|
|
|
@pytest.mark.parametrize('failed_write', [1, 2])
|
|
@pytest.mark.parametrize('scheduled', [False, True])
|
|
def test_final_receipt_write_failure_is_durable_and_propagates(project_service, monkeypatch, failed_write, scheduled):
|
|
service, _, data, ids, hashes = project_service
|
|
run_id = 'd' * 32
|
|
schedule = ScheduleStore(data).save({'task_ids': ids, 'priority': 'normal',
|
|
'not_before': '2026-01-01T00:00:00+00:00'}, hashes) if scheduled else None
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *a, **kw:
|
|
TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False))
|
|
finish = RunJournal.finish
|
|
calls = 0
|
|
def broken_finish(self, key, receipt):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == failed_write:
|
|
fail('receipt boundary')
|
|
return finish(self, key, receipt)
|
|
monkeypatch.setattr(RunJournal, 'finish', broken_finish)
|
|
with pytest.raises(OSError, match='receipt boundary'):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, schedule=schedule)
|
|
row = RunJournal(data).get(run_id)
|
|
assert row['status'] == 'interrupted'
|
|
assert [entry['state'] for entry in row['result']['entries']] == ['completed'] * len(ids)
|
|
assert row['result']['code'] == 'receipt_write_error'
|
|
if scheduled:
|
|
after = ScheduleStore(data).get(schedule['schedule_id'])
|
|
assert after['status'] == 'interrupted' and after['run_id'] == run_id
|
|
|
|
|
|
@pytest.mark.parametrize('damage', ['scope', 'unknown'])
|
|
def test_saved_retry_with_changed_scope_or_unknown_proof_records_observation_without_replay(rule_run, monkeypatch, damage):
|
|
case = rule_run(scheduled=True)
|
|
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
|
|
selected = service.reserve_problem_retry(task_id, source='dsh')
|
|
service.prepare_failed_quick_run(task_id, operation_keys=selected)
|
|
prepared = service.task_context(task_id)
|
|
p = prepared.progress
|
|
expected = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number,
|
|
'part_label' if prepared.task.copy_part_number_to_value else 'preserve')
|
|
execute = service.execute_quick_run_queue
|
|
path = root / 'task-queues' / (run_id + '.json')
|
|
def damage_after_save(*args, **kwargs):
|
|
outcome = execute(*args, **kwargs)
|
|
assert service.task_context(task_id).progress.saved_items == 2
|
|
if damage == 'scope':
|
|
state = json.loads(path.read_text(encoding='utf-8'))
|
|
state['task_ids'] = ['f' * 32]
|
|
path.write_text(json.dumps(state), encoding='utf-8')
|
|
else:
|
|
case['store']._connection.execute('UPDATE execution_operations SET state=? WHERE operation_key=?',
|
|
('uncertain', selected[0]))
|
|
fail('saved outcome now uncertain')
|
|
return outcome
|
|
monkeypatch.setattr(service, 'execute_quick_run_queue', damage_after_save)
|
|
with pytest.raises((TaskCenterError, OSError)):
|
|
execute_incident_retry(service, root, task_id, run_id, expected_item=expected,
|
|
operation_keys=selected, stop_event=Event())
|
|
journal = RunJournal(root)
|
|
row = journal.get(run_id)
|
|
assert row['status'] == 'interrupted'
|
|
assert row['result']['tasks'][0]['saved_items'] == 1, 'Unsafe evidence cannot replace the authorized baseline'
|
|
if damage == 'scope':
|
|
assert row['result']['retry_observation']['saved_items'] == 2
|
|
assert row['result']['retry_observation']['task_id'] == task_id
|
|
inspected = journal.inspect(service, run_id)
|
|
assert not inspected['resume_ready']
|
|
with pytest.raises(TaskCenterError):
|
|
execute_confirmed_run(service, root, [task_id], preflight(service, [task_id], allow_completed=True),
|
|
Event(), run_id, resume=inspected)
|
|
assert case['calls'] == ['OLD-OK', 'OLD-BAD', 'OLD-BAD']
|
|
schedule = ScheduleStore(root).get(case['schedule']['schedule_id'])
|
|
assert schedule['status'] == 'interrupted' and schedule['run_id'] == run_id
|
|
|
|
|
|
def test_scheduled_resume_terminal_publication_failure_closes_claim(project_service, monkeypatch):
|
|
service, _, data, ids, hashes = project_service
|
|
run_id = 'e' * 32
|
|
schedules = ScheduleStore(data)
|
|
schedule = schedules.save({'task_ids': ids, 'priority': 'normal',
|
|
'not_before': '2026-01-01T00:00:00+00:00'}, hashes)
|
|
schedules.claim(schedule['schedule_id'], run_id=run_id)
|
|
journal = RunJournal(data)
|
|
journal.begin(run_id, ids, hashes, schedule_id=schedule['schedule_id'])
|
|
checkpoint(data, run_id, ids, hashes)
|
|
journal.finish(run_id, {'status': 'paused', 'entries': []})
|
|
resume = journal.inspect(service, run_id)
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *a, **kw:
|
|
TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False))
|
|
finish, calls = ScheduleStore.finish, []
|
|
def broken_once(self, *args, **kwargs):
|
|
calls.append(True)
|
|
if len(calls) == 1:
|
|
fail('schedule publication failed')
|
|
return finish(self, *args, **kwargs)
|
|
monkeypatch.setattr(ScheduleStore, 'finish', broken_once)
|
|
with pytest.raises(OSError, match='schedule publication failed'):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
|
|
after = schedules.get(schedule['schedule_id'])
|
|
assert after['status'] == 'interrupted' and after['run_id'] == run_id
|
|
assert journal.get(run_id)['status'] == 'completed', 'Known executor receipt stays durable'
|
|
|
|
|
|
@pytest.mark.parametrize('operation', ['begin', 'resume', 'finish'])
|
|
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
|
|
def test_running_producer_returns_committed_record_without_fallible_tail_read(project_service, monkeypatch, operation, error_type):
|
|
_, _, data, ids, hashes = project_service
|
|
run_id = '1' * 32
|
|
journal = RunJournal(data)
|
|
if operation in {'resume', 'finish'}:
|
|
journal.begin(run_id, ids, hashes)
|
|
if operation == 'resume':
|
|
journal.finish(run_id, {'status': 'paused', 'entries': []})
|
|
read, failures = RunJournal.get, []
|
|
def unavailable_once(self, key):
|
|
row = read(self, key)
|
|
if row and row['status'] == ('completed' if operation == 'finish' else 'running') and not failures:
|
|
failures.append(True)
|
|
raise error_type('post-commit read unavailable')
|
|
return row
|
|
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
|
|
returned = (journal.begin(run_id, ids, hashes) if operation == 'begin' else journal.resume(run_id)
|
|
if operation == 'resume' else journal.finish(run_id, {'status': 'completed', 'entries': []}))
|
|
assert returned == read(RunJournal(data), run_id)
|
|
assert returned['status'] == ('completed' if operation == 'finish' else 'running')
|
|
assert not failures, 'A committed mutation must not need a second connection/read to acknowledge it'
|
|
|
|
|
|
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
|
|
def test_retry_producer_returns_real_intent_without_post_commit_read(rule_run, monkeypatch, error_type):
|
|
case = rule_run()
|
|
service, task_id, run_id, root = (case[key] for key in ('service', 'task_id', 'run_id', 'root'))
|
|
selected = service.reserve_problem_retry(task_id, source='dsh')
|
|
service.prepare_failed_quick_run(task_id, operation_keys=selected)
|
|
read, failures = RunJournal.get, []
|
|
def unavailable_once(self, key):
|
|
row = read(self, key)
|
|
if row and row['status'] == 'running' and not failures:
|
|
failures.append(True)
|
|
raise error_type('post-commit retry read unavailable')
|
|
return row
|
|
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
|
|
returned = RunJournal(root).begin_retry(service, run_id, task_id, selected)
|
|
assert returned == read(RunJournal(root), run_id)
|
|
assert returned['status'] == 'running' and returned['result']['retry']['operation_keys'] == sorted(selected)
|
|
assert case['calls'] == ['OLD-OK', 'OLD-BAD']
|
|
assert not failures
|
|
|
|
|
|
@pytest.mark.parametrize('error_type', [OSError, sqlite3.OperationalError])
|
|
def test_post_execution_read_failure_cannot_lose_known_outcome(project_service, monkeypatch, error_type):
|
|
service, _, data, ids, _ = project_service
|
|
run_id = '2' * 32
|
|
report = {ids[0]: {'status': 'verified', 'verified': True, 'report_path': 'known-save.json'}}
|
|
read, failures, executed = RunJournal.get, [], []
|
|
def executor(*args, **kwargs):
|
|
executed.append(True)
|
|
return TaskQueueResult(run_id, tuple(TaskQueueEntry(key, 'completed') for key in ids), False,
|
|
netlist_verification=report)
|
|
def unavailable_once(self, key):
|
|
if executed and not failures:
|
|
failures.append(True)
|
|
raise error_type('post-execution read unavailable')
|
|
return read(self, key)
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', executor)
|
|
monkeypatch.setattr(RunJournal, 'get', unavailable_once)
|
|
try:
|
|
receipt = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id)
|
|
except (OSError, sqlite3.OperationalError):
|
|
receipt = read(RunJournal(data), run_id)['result']
|
|
row = read(RunJournal(data), run_id)
|
|
assert row['status'] in {'completed', 'interrupted'}
|
|
assert [entry['state'] for entry in row['result']['entries']] == ['completed'] * len(ids)
|
|
assert row['result']['netlist_verification'] == report
|
|
assert len(executed) == 1
|