195 lines
9.5 KiB
Python
195 lines
9.5 KiB
Python
"""Original queue recovery across a fatal receipt-write boundary."""
|
|
from pathlib import Path
|
|
import subprocess
|
|
from threading import Event
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.dsh.execution_runs import execute_confirmed_run
|
|
from dx_productivity.dsh.run_journal import RunJournal
|
|
from dx_productivity.dsh.task_lifecycle import preflight
|
|
from test_run_journal import project_service
|
|
from test_run_journal import checkpoint, evidence
|
|
|
|
|
|
def _pause_then_fail_completed_checkpoint(project_service, monkeypatch, *, before_failure=None):
|
|
from dx_productivity.trial_task_center import task_queue
|
|
service, _, data, ids, _ = project_service
|
|
calls = []
|
|
stop = Event()
|
|
|
|
def worker(command, timeout):
|
|
queue, job = Path(command[3]), command[-1]
|
|
calls.append(next(key for key in ids if key in queue.parts))
|
|
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
|
|
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
|
|
if len(calls) == 1:
|
|
stop.set()
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
|
|
for task_id in ids:
|
|
preparer = service.for_project_task(task_id)._quick_run_preparer
|
|
preparer._process_runner = worker
|
|
preparer._cscript_resolver = lambda: Path('cscript.exe')
|
|
run_id = 'f' * 32
|
|
first = execute_confirmed_run(service, data, ids, preflight(service, ids), stop, run_id)
|
|
assert first['status'] == 'paused' and calls == ids[:1]
|
|
original = RunJournal(data).inspect(service, run_id)
|
|
save = task_queue._save
|
|
|
|
def fail_completed(path, state):
|
|
if state['entries'][ids[1]]['state'] == 'completed':
|
|
if before_failure:
|
|
before_failure(service, ids)
|
|
raise OSError('controlled completed checkpoint publication failure')
|
|
save(path, state)
|
|
|
|
with monkeypatch.context() as patch:
|
|
patch.setattr(task_queue, '_save', fail_completed)
|
|
with pytest.raises(OSError, match='checkpoint publication failure'):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
|
Event(), run_id, resume=original)
|
|
assert calls == ids[:2]
|
|
return run_id, first, calls
|
|
|
|
|
|
def test_resume_after_verified_progress_and_checkpoint_io_failure_runs_each_task_once(project_service, monkeypatch):
|
|
service, _, data, ids, _ = project_service
|
|
run_id, first, calls = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
|
|
restarted = RunJournal(data).inspect(service, run_id)
|
|
assert restarted['status'] == 'interrupted'
|
|
assert restarted['checkpoint_states'][ids[1]] == 'running'
|
|
assert restarted['resume_ready'], restarted['blockers']
|
|
result = restarted['result']
|
|
old_terminal = next(row for row in result['tasks'] if row['task_id'] == ids[0])
|
|
assert old_terminal == next(row for row in first['tasks'] if row['task_id'] == ids[0])
|
|
advanced = next(row for row in result['tasks'] if row['task_id'] == ids[1])
|
|
assert advanced['no_match_items'] == 1 and advanced['queue_complete'] is True
|
|
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
|
Event(), run_id, resume=restarted)
|
|
assert last['status'] == 'completed' and last['queue_id'] == run_id
|
|
assert calls == ids
|
|
assert all(row['no_match_items'] == 1 and row['failed_items'] == 0 for row in last['tasks'])
|
|
|
|
|
|
@pytest.mark.parametrize('task_index', [0, 1, 2])
|
|
def test_soft_failure_snapshot_does_not_allow_evidence_changes_while_stopped(project_service, monkeypatch, task_index):
|
|
service, _, data, ids, _ = project_service
|
|
run_id, _, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
|
|
if task_index == 2:
|
|
evidence(service, ids[task_index])
|
|
else:
|
|
root = service.for_project_task(ids[task_index])._quick_run_preparer.data_root / 'quick-run'
|
|
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
|
|
restarted = RunJournal(data).inspect(service, run_id)
|
|
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['old_terminal', 'failed_result', 'uncertain_result'])
|
|
def test_checkpoint_failure_does_not_adopt_unsafe_evidence_as_new_baseline(project_service, monkeypatch, fault):
|
|
service, _, data, ids, _ = project_service
|
|
|
|
def corrupt(service, ids):
|
|
if fault == 'old_terminal':
|
|
root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
|
|
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
|
|
else:
|
|
evidence(service, ids[1], 'batch_failed' if fault == 'failed_result' else 'batch_saved_unverified', '')
|
|
|
|
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, before_failure=corrupt)
|
|
restarted = RunJournal(data).inspect(service, run_id)
|
|
assert not restarted['resume_ready']
|
|
assert restarted['result']['tasks'] == first['tasks']
|
|
assert 'evidence_changed' in restarted['blockers']
|
|
|
|
|
|
def test_failed_exception_readback_keeps_original_error_and_baseline(project_service, monkeypatch):
|
|
service, _, data, _, _ = project_service
|
|
inspect = RunJournal.inspect
|
|
failing = False
|
|
|
|
def unavailable(self, *args, **kwargs):
|
|
if failing:
|
|
raise RuntimeError('controlled readback failure')
|
|
return inspect(self, *args, **kwargs)
|
|
|
|
def start_failing(_service, _ids):
|
|
nonlocal failing
|
|
failing = True
|
|
|
|
with monkeypatch.context() as patch:
|
|
patch.setattr(RunJournal, 'inspect', unavailable)
|
|
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch,
|
|
before_failure=start_failing)
|
|
restarted = RunJournal(data).inspect(service, run_id)
|
|
assert restarted['result']['tasks'] == first['tasks']
|
|
assert restarted['result']['entries'] == first['entries']
|
|
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
|
|
|
|
|
|
def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch):
|
|
service, _, data, ids, _ = project_service
|
|
calls = []
|
|
active = {'stop': Event(), 'pause': True}
|
|
def worker(command, timeout):
|
|
queue, job = Path(command[3]), command[-1]
|
|
calls.append(next(key for key in ids if key in queue.parts))
|
|
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
|
|
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
|
|
if active['pause']:
|
|
active['stop'].set()
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
for task_id in ids:
|
|
preparer = service.for_project_task(task_id)._quick_run_preparer
|
|
preparer._process_runner = worker
|
|
preparer._cscript_resolver = lambda: Path('cscript.exe')
|
|
run_id = 'd' * 32
|
|
first = execute_confirmed_run(service, data, ids, preflight(service, ids), active['stop'], run_id)
|
|
assert first['status'] == 'paused' and calls == ids[:1]
|
|
original = RunJournal(data).inspect(service, run_id)
|
|
active['stop'] = Event()
|
|
class FatalExit(BaseException):
|
|
pass
|
|
with monkeypatch.context() as patch:
|
|
def fatal_finish(*args, **kwargs):
|
|
raise FatalExit('exit after checkpoint, before final receipt')
|
|
patch.setattr(RunJournal, 'finish', fatal_finish)
|
|
with pytest.raises(FatalExit):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
|
active['stop'], run_id, resume=original)
|
|
assert calls == ids[:2]
|
|
restarted = RunJournal(data).inspect(service, run_id)
|
|
assert restarted['status'] == 'running' and restarted['resume_ready'], restarted['blockers']
|
|
assert restarted['remaining_task_ids'] == ids[2:]
|
|
active.update(stop=Event(), pause=False)
|
|
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
|
active['stop'], run_id, resume=restarted)
|
|
assert last['status'] == 'completed' and last['queue_id'] == run_id
|
|
assert calls == ids
|
|
assert all(task['saved_items'] == 0 and task['no_match_items'] == 1 for task in last['tasks'])
|
|
|
|
|
|
def test_resume_soft_failure_retains_the_last_evidence_baseline(project_service, monkeypatch):
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
run_id = 'e' * 32
|
|
journal.begin(run_id, ids, hashes)
|
|
checkpoint(data, run_id, ids, hashes)
|
|
prior = {'status': 'paused', 'tasks': [{'task_id': ids[0], 'saved_items': 0, 'failed_items': 0,
|
|
'no_match_items': 0, 'queue_complete': False, 'interrupted': False}],
|
|
'entries': [{'task_id': key, 'state': 'paused' if key == ids[0] else 'pending'} for key in ids]}
|
|
journal.finish(run_id, prior)
|
|
resume = journal.inspect(service, run_id)
|
|
def unavailable(*args, **kwargs):
|
|
raise OSError('controlled queue failure before progress')
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', unavailable)
|
|
with pytest.raises(OSError):
|
|
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
|
|
result = journal.get(run_id)['result']
|
|
assert result['status'] == 'interrupted'
|
|
assert result['tasks'] == prior['tasks'] and result['entries'] == prior['entries']
|
|
evidence(service, ids[0])
|
|
inspected = journal.inspect(service, run_id)
|
|
assert inspected['diff']['evidence_baseline_available']
|
|
assert 'evidence_changed' in inspected['blockers']
|