Files
dxdesigner-part-replacement/tests/dsh/test_execution_recovery.py

196 lines
9.6 KiB
Python

"""Original queue recovery across a fatal receipt-write boundary."""
from pathlib import Path
import subprocess
from threading import Event
import pytest
from dx_productivity.dsh.execution_runs import execute_confirmed_run
from dx_productivity.dsh.run_journal import RunJournal
from dx_productivity.dsh.task_lifecycle import preflight
from test_run_journal import project_service
from test_run_journal import checkpoint, evidence
def _pause_then_fail_completed_checkpoint(project_service, monkeypatch, *, before_failure=None):
from dx_productivity.trial_task_center import task_queue
service, _, data, ids, _ = project_service
calls = []
stop = Event()
def worker(command, timeout):
# Windows may pass a short compatibility path to the native worker.
queue, job = Path(command[3]).resolve(), command[-1]
calls.append(next(key for key in ids if key in queue.parts))
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
if len(calls) == 1:
stop.set()
return subprocess.CompletedProcess(command, 0, '', '')
for task_id in ids:
preparer = service.for_project_task(task_id)._quick_run_preparer
preparer._process_runner = worker
preparer._cscript_resolver = lambda: Path('cscript.exe')
run_id = 'f' * 32
first = execute_confirmed_run(service, data, ids, preflight(service, ids), stop, run_id)
assert first['status'] == 'paused' and calls == ids[:1]
original = RunJournal(data).inspect(service, run_id)
save = task_queue._save
def fail_completed(path, state):
if state['entries'][ids[1]]['state'] == 'completed':
if before_failure:
before_failure(service, ids)
raise OSError('controlled completed checkpoint publication failure')
save(path, state)
with monkeypatch.context() as patch:
patch.setattr(task_queue, '_save', fail_completed)
with pytest.raises(OSError, match='checkpoint publication failure'):
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
Event(), run_id, resume=original)
assert calls == ids[:2]
return run_id, first, calls
def test_resume_after_verified_progress_and_checkpoint_io_failure_runs_each_task_once(project_service, monkeypatch):
service, _, data, ids, _ = project_service
run_id, first, calls = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
restarted = RunJournal(data).inspect(service, run_id)
assert restarted['status'] == 'interrupted'
assert restarted['checkpoint_states'][ids[1]] == 'running'
assert restarted['resume_ready'], restarted['blockers']
result = restarted['result']
old_terminal = next(row for row in result['tasks'] if row['task_id'] == ids[0])
assert old_terminal == next(row for row in first['tasks'] if row['task_id'] == ids[0])
advanced = next(row for row in result['tasks'] if row['task_id'] == ids[1])
assert advanced['no_match_items'] == 1 and advanced['queue_complete'] is True
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
Event(), run_id, resume=restarted)
assert last['status'] == 'completed' and last['queue_id'] == run_id
assert calls == ids
assert all(row['no_match_items'] == 1 and row['failed_items'] == 0 for row in last['tasks'])
@pytest.mark.parametrize('task_index', [0, 1, 2])
def test_soft_failure_snapshot_does_not_allow_evidence_changes_while_stopped(project_service, monkeypatch, task_index):
service, _, data, ids, _ = project_service
run_id, _, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
if task_index == 2:
evidence(service, ids[task_index])
else:
root = service.for_project_task(ids[task_index])._quick_run_preparer.data_root / 'quick-run'
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
restarted = RunJournal(data).inspect(service, run_id)
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
@pytest.mark.parametrize('fault', ['old_terminal', 'failed_result', 'uncertain_result'])
def test_checkpoint_failure_does_not_adopt_unsafe_evidence_as_new_baseline(project_service, monkeypatch, fault):
service, _, data, ids, _ = project_service
def corrupt(service, ids):
if fault == 'old_terminal':
root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
else:
evidence(service, ids[1], 'batch_failed' if fault == 'failed_result' else 'batch_saved_unverified', '')
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, before_failure=corrupt)
restarted = RunJournal(data).inspect(service, run_id)
assert not restarted['resume_ready']
assert restarted['result']['tasks'] == first['tasks']
assert 'evidence_changed' in restarted['blockers']
def test_failed_exception_readback_keeps_original_error_and_baseline(project_service, monkeypatch):
service, _, data, _, _ = project_service
inspect = RunJournal.inspect
failing = False
def unavailable(self, *args, **kwargs):
if failing:
raise RuntimeError('controlled readback failure')
return inspect(self, *args, **kwargs)
def start_failing(_service, _ids):
nonlocal failing
failing = True
with monkeypatch.context() as patch:
patch.setattr(RunJournal, 'inspect', unavailable)
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch,
before_failure=start_failing)
restarted = RunJournal(data).inspect(service, run_id)
assert restarted['result']['tasks'] == first['tasks']
assert restarted['result']['entries'] == first['entries']
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch):
service, _, data, ids, _ = project_service
calls = []
active = {'stop': Event(), 'pause': True}
def worker(command, timeout):
queue, job = Path(command[3]).resolve(), command[-1]
calls.append(next(key for key in ids if key in queue.parts))
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
if active['pause']:
active['stop'].set()
return subprocess.CompletedProcess(command, 0, '', '')
for task_id in ids:
preparer = service.for_project_task(task_id)._quick_run_preparer
preparer._process_runner = worker
preparer._cscript_resolver = lambda: Path('cscript.exe')
run_id = 'd' * 32
first = execute_confirmed_run(service, data, ids, preflight(service, ids), active['stop'], run_id)
assert first['status'] == 'paused' and calls == ids[:1]
original = RunJournal(data).inspect(service, run_id)
active['stop'] = Event()
class FatalExit(BaseException):
pass
with monkeypatch.context() as patch:
def fatal_finish(*args, **kwargs):
raise FatalExit('exit after checkpoint, before final receipt')
patch.setattr(RunJournal, 'finish', fatal_finish)
with pytest.raises(FatalExit):
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
active['stop'], run_id, resume=original)
assert calls == ids[:2]
restarted = RunJournal(data).inspect(service, run_id)
assert restarted['status'] == 'running' and restarted['resume_ready'], restarted['blockers']
assert restarted['remaining_task_ids'] == ids[2:]
active.update(stop=Event(), pause=False)
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
active['stop'], run_id, resume=restarted)
assert last['status'] == 'completed' and last['queue_id'] == run_id
assert calls == ids
assert all(task['saved_items'] == 0 and task['no_match_items'] == 1 for task in last['tasks'])
def test_resume_soft_failure_retains_the_last_evidence_baseline(project_service, monkeypatch):
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
run_id = 'e' * 32
journal.begin(run_id, ids, hashes)
checkpoint(data, run_id, ids, hashes)
prior = {'status': 'paused', 'tasks': [{'task_id': ids[0], 'saved_items': 0, 'failed_items': 0,
'no_match_items': 0, 'queue_complete': False, 'interrupted': False}],
'entries': [{'task_id': key, 'state': 'paused' if key == ids[0] else 'pending'} for key in ids]}
journal.finish(run_id, prior)
resume = journal.inspect(service, run_id)
def unavailable(*args, **kwargs):
raise OSError('controlled queue failure before progress')
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', unavailable)
with pytest.raises(OSError):
execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), run_id, resume=resume)
result = journal.get(run_id)['result']
assert result['status'] == 'interrupted'
assert result['tasks'] == prior['tasks'] and result['entries'] == prior['entries']
evidence(service, ids[0])
inspected = journal.inspect(service, run_id)
assert inspected['diff']['evidence_baseline_available']
assert 'evidence_changed' in inspected['blockers']