Files
dxdesigner-part-replacement/tests/trial_task_center/test_recovery.py
T

289 lines
14 KiB
Python

import json
from pathlib import Path
import subprocess
from types import SimpleNamespace
import pytest
from dx_productivity.trial_task_center.errors import TaskCenterError
from dx_productivity.trial_task_center.native_helper import read_snapshot, transaction_binding
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer, _read_commands, _read_status_details
from dx_productivity.trial_task_center.recovery import _is_link_or_reparse, _recovery_hashes, recover
def test_link_detection_does_not_require_path_is_junction():
path = SimpleNamespace(lstat=lambda: SimpleNamespace(st_mode=0, st_file_attributes=0))
assert not _is_link_or_reparse(path)
def test_locked_recovery_files_return_stable_recovery_error(tmp_path, monkeypatch):
import dx_productivity.trial_task_center.recovery as recovery_module
def locked(_root):
raise PermissionError('locked')
monkeypatch.setattr(recovery_module, '_hashes', locked)
with pytest.raises(TaskCenterError) as raised:
_recovery_hashes(tmp_path)
assert raised.value.code == 'quick_run_recovery'
assert '占用' in raised.value.message
@pytest.fixture
def interrupted(tmp_path):
project = tmp_path / 'project' / 'one.prj'
project.parent.mkdir()
project.write_text('project')
script = tmp_path / 'runtime.vbs'
script.write_text('fixture runtime')
root = tmp_path / 'data'
preparer = QuickRunPreparer(root, script_source=script, run_token_factory=lambda: 'first',
cscript_resolver=lambda: Path('cscript.exe'))
projects = (SimpleNamespace(path=project, selected=True, available=True),)
rules = (SimpleNamespace(old_part_number='OLD', new_part_number='NEW', enabled=True),)
handoff = preparer.prepare('task', projects, rules, copy_part_number_to_value=True)
job = _read_commands(handoff.command_path)[0][1]
snapshot = handoff.command_path.parent / f'snapshot-{job}.tsv'
snapshot.write_text(f'M\t{project}\tOLD\tNEW\tA:1\tpart_label\nC\tuid\tR1\t0\t0\t0\t1\tpins\n', encoding='utf-8')
metadata, components = read_snapshot(snapshot)
report = {'stage': 'native_replace_verified', 'transaction': transaction_binding(
snapshot, metadata, components, ['library', 'NEW', 'symbol', '', 'cap', 'label'], 'symbol')}
snapshot.with_suffix('.tsv.native.json').write_text(json.dumps(report), encoding='utf-8')
handoff.status_path.write_text('job_id\tstate\tcode\tmessage\n' + job + '\tawaiting_native_replace\t\tstarted\n', encoding='utf-8')
lock = handoff.command_path.parent / 'uncertain.flag'
lock.write_text('old timeout')
return SimpleNamespace(preparer=preparer, root=root, projects=projects, rules=rules, snapshot=snapshot,
job=job, handoff=handoff, lock=lock, report=report)
def invoke(fixture, **kwargs):
return recover(fixture.preparer, 'task', fixture.projects, fixture.rules,
copy_part_number_to_value=True, confirmed=True, **kwargs)
def worker_result(fixture, state='restored_ok', job=None, code=0):
def run(command, timeout):
worker = Path(command[3])
assert worker != fixture.lock.parent
assert (worker / 'recovery.request').read_text() == fixture.job
assert timeout == 150
assert fixture.lock.exists()
(worker / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n' + (job or fixture.job) + '\t' + state + '\t\tverified\n')
return subprocess.CompletedProcess(command, code, '', '')
return run
@pytest.mark.parametrize('hard_failure', [False, True])
def test_legacy_interruption_without_flag_can_be_reconciled_only_with_bound_evidence(interrupted, hard_failure):
interrupted.lock.unlink()
if hard_failure:
with interrupted.handoff.status_path.open('a', encoding='utf-8') as stream:
stream.write(interrupted.job + '\thard_failure\t1\tnative write failed\n')
assert interrupted.preparer.has_interrupted('task')
assert not interrupted.preparer.has_interrupted('other-task')
interrupted.preparer._process_runner = worker_result(interrupted, 'retry_ready')
result = invoke(interrupted)
assert result.state == 'retry_ready'
assert not interrupted.lock.exists()
assert not interrupted.preparer.has_interrupted('task')
def test_preflight_failure_without_transaction_does_not_offer_unsafe_recovery(interrupted):
interrupted.lock.unlink()
with interrupted.handoff.status_path.open('a', encoding='utf-8') as stream:
stream.write(interrupted.job + '\thard_failure\t1\tmissing target\n')
interrupted.snapshot.with_suffix('.tsv.native.json').write_text('{"stage":"preflight","ok":false}', encoding='utf-8')
assert not interrupted.preparer.has_interrupted('task')
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert not interrupted.lock.exists()
def test_recovery_requires_explicit_confirmation(interrupted):
with pytest.raises(TaskCenterError, match='quick_run_recovery_confirmation'):
recover(interrupted.preparer, 'task', interrupted.projects, interrupted.rules)
assert interrupted.lock.exists()
def test_verified_recovery_preserves_evidence_and_commits_once(interrupted):
interrupted.preparer._process_runner = worker_result(interrupted)
snapshot = interrupted.snapshot.read_bytes()
result = invoke(interrupted)
assert result.state == 'restored_ok'
assert result.next_project_path is None
assert not interrupted.lock.exists()
assert interrupted.snapshot.read_bytes() == snapshot
assert _read_status_details(interrupted.handoff.status_path)[-1][1] == 'restored_ok'
audit = next((interrupted.root / 'quick-run-history').glob('recovery-*'))
assert (audit / 'resolved-uncertain.flag').read_text() == 'old timeout'
assert (audit / 'evidence-before' / interrupted.snapshot.name).read_bytes() == snapshot
with pytest.raises(TaskCenterError):
invoke(interrupted)
def test_r12_part_number_queue_keeps_recorded_policy_during_recovery(interrupted):
command_path = interrupted.handoff.command_path
lines = [line.split('\t') for line in command_path.read_text(encoding='utf-8').splitlines()]
for column in ('value_mode', 'base_value_mode'):
lines[1][lines[0].index(column)] = 'part_number'
command_path.write_text('\n'.join('\t'.join(line) for line in lines) + '\n', encoding='utf-8')
interrupted.snapshot.write_text(
interrupted.snapshot.read_text(encoding='utf-8').replace('\tpart_label\n', '\tpart_number\n'),
encoding='utf-8',
)
metadata, components = read_snapshot(interrupted.snapshot)
report = {'stage': 'native_replace_verified', 'transaction': transaction_binding(
interrupted.snapshot, metadata, components,
['library', 'NEW', 'symbol', '', 'cap', 'label'], 'symbol')}
interrupted.snapshot.with_suffix('.tsv.native.json').write_text(json.dumps(report), encoding='utf-8')
def worker(command, timeout):
worker_root = Path(command[3])
assert '\tpart_number\t' in (worker_root / 'command.tsv').read_text(encoding='utf-8')
return worker_result(interrupted)(command, timeout)
interrupted.preparer._process_runner = worker
result = invoke(interrupted)
assert result.state == 'restored_ok'
assert result.next_project_path is None
assert interrupted.preparer.progress(
'task', interrupted.projects, interrupted.rules, copy_part_number_to_value=True
).queue_complete
def test_page_addressed_recovery_keeps_the_original_sheet_binding(interrupted):
project = interrupted.projects[0].path
interrupted.handoff.command_path.write_text(
'task_id\tjob_id\tproject_path\tschematic_name\tsheet_number\told_part\tnew_part\tvalue_mode\n'
f'task\t{interrupted.job}\t{project}\tA\t1\tOLD\tNEW\tpart_label\n',
encoding='utf-8',
)
(interrupted.handoff.command_path.parent / 'sheets.tsv').write_text(
'project_path\tschematic_name\tsheet_number\n'
f'{project}\tA\t1\n',
encoding='utf-8',
)
def worker(command, timeout):
worker_root = Path(command[3])
header = (worker_root / 'command.tsv').read_text(encoding='utf-8').splitlines()[0].split('\t')
row = (worker_root / 'command.tsv').read_text(encoding='utf-8').splitlines()[1].split('\t')
assert row[header.index('schematic_name')] == 'A'
assert row[header.index('sheet_number')] == '1'
assert (worker_root / 'sheets.tsv').is_file()
(worker_root / 'status.tsv').write_text(
f'job_id\tstate\tcode\tmessage\n{interrupted.job}\trestored_ok\t\tverified\n',
encoding='utf-8',
)
return subprocess.CompletedProcess(command, 0, '', '')
interrupted.preparer._process_runner = worker
result = invoke(interrupted)
assert result.state == 'restored_ok'
assert result.next_project_path is None
def test_unsaved_recovery_creates_fresh_job_without_replacing(interrupted):
interrupted.preparer._process_runner = worker_result(interrupted, 'retry_ready')
result = invoke(interrupted)
assert result.state == 'retry_ready'
assert result.next_project_path == interrupted.projects[0].path
assert _read_commands(interrupted.handoff.command_path)[0][1] != interrupted.job
assert _read_status_details(interrupted.handoff.status_path) == ()
assert interrupted.snapshot.exists()
@pytest.mark.parametrize(('state', 'job', 'code'), [('hard_failure', None, 1), ('restored_ok', 'foreign', 0),
('awaiting_native_replace', None, 0), ('restored_ok', None, 2)])
def test_uncorrelated_or_failed_recovery_never_unlocks(interrupted, state, job, code):
before = interrupted.handoff.status_path.read_bytes()
interrupted.preparer._process_runner = worker_result(interrupted, state, job, code)
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert interrupted.lock.exists()
assert interrupted.handoff.status_path.read_bytes() == before
def test_recovery_timeout_keeps_old_lock(interrupted):
def timeout(command, seconds):
raise subprocess.TimeoutExpired(command, seconds)
interrupted.preparer._process_runner = timeout
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert interrupted.lock.read_text() == 'old timeout'
def test_snapshot_binding_mismatch_prevents_worker_start(interrupted):
interrupted.snapshot.write_text(interrupted.snapshot.read_text() + '\n')
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert interrupted.lock.exists()
assert not (interrupted.root / 'quick-run-history').exists()
def test_concurrent_evidence_change_does_not_commit(interrupted):
worker = worker_result(interrupted)
def change(command, seconds):
result = worker(command, seconds)
interrupted.lock.write_text('new protection')
return result
interrupted.preparer._process_runner = change
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert interrupted.lock.read_text() == 'new protection'
def test_other_pending_job_prevents_global_unlock(interrupted):
with interrupted.handoff.status_path.open('a') as stream:
stream.write('other-job\thard_failure\t\tunknown\n')
with pytest.raises(TaskCenterError):
invoke(interrupted)
assert interrupted.lock.exists()
def test_failed_commit_restores_status_and_lock(interrupted, monkeypatch):
from dx_productivity.trial_task_center import recovery
interrupted.preparer._process_runner = worker_result(interrupted)
original = interrupted.handoff.status_path.read_bytes()
real_replace = recovery.os.replace
def fail_unlock(source, target):
if Path(source) == interrupted.lock:
raise PermissionError('locked')
return real_replace(source, target)
monkeypatch.setattr(recovery.os, 'replace', fail_unlock)
with pytest.raises(PermissionError):
invoke(interrupted)
assert interrupted.lock.exists()
assert interrupted.handoff.status_path.read_bytes() == original
@pytest.mark.parametrize('initial', ['ready', 'paused', 'failed'])
def test_service_recovery_updates_queue_and_database(interrupted, initial):
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.models import TaskStatus
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
store = TaskStore(interrupted.root / 'task-center.db')
service = TaskCenterService(store, configure_logging(interrupted.root), quick_run_preparer=interrupted.preparer)
task = service.create_task('Interrupted')
service.set_copy_part_number_to_value(task.task_id, True)
projects = service.scan_projects(task.task_id, interrupted.projects[0].path.parent)
service.set_project_selected(projects[0].project_id, True)
service.add_rule(task.task_id, 'OLD', 'NEW')
command_path = interrupted.handoff.command_path
command_path.write_text(command_path.read_text().replace('task\t', task.task_id + '\t'), encoding='utf-8')
if store.get_task(task.task_id).status is TaskStatus.DRAFT:
store.set_task_status(task.task_id, TaskStatus.READY)
if initial == 'paused':
store.set_task_status(task.task_id, TaskStatus.RUNNING)
store.set_task_status(task.task_id, TaskStatus.PAUSED)
elif initial == 'failed':
store.set_task_status(task.task_id, TaskStatus.FAILED)
interrupted.preparer._process_runner = worker_result(interrupted)
result = service.recover_quick_run(task.task_id, confirmed=True)
assert result.state == 'restored_ok'
assert store.get_task(task.task_id).status is TaskStatus.COMPLETED
assert service.quick_run_progress(task.task_id).queue_complete
assert not service.has_interrupted_quick_run(task.task_id)