289 lines
14 KiB
Python
289 lines
14 KiB
Python
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
from dx_productivity.trial_task_center.native_helper import read_snapshot, transaction_binding
|
|
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer, _read_commands, _read_status_details
|
|
from dx_productivity.trial_task_center.recovery import _is_link_or_reparse, _recovery_hashes, recover
|
|
|
|
|
|
def test_link_detection_does_not_require_path_is_junction():
|
|
path = SimpleNamespace(lstat=lambda: SimpleNamespace(st_mode=0, st_file_attributes=0))
|
|
assert not _is_link_or_reparse(path)
|
|
|
|
|
|
def test_locked_recovery_files_return_stable_recovery_error(tmp_path, monkeypatch):
|
|
import dx_productivity.trial_task_center.recovery as recovery_module
|
|
|
|
def locked(_root):
|
|
raise PermissionError('locked')
|
|
|
|
monkeypatch.setattr(recovery_module, '_hashes', locked)
|
|
with pytest.raises(TaskCenterError) as raised:
|
|
_recovery_hashes(tmp_path)
|
|
assert raised.value.code == 'quick_run_recovery'
|
|
assert '占用' in raised.value.message
|
|
|
|
|
|
@pytest.fixture
|
|
def interrupted(tmp_path):
|
|
project = tmp_path / 'project' / 'one.prj'
|
|
project.parent.mkdir()
|
|
project.write_text('project')
|
|
script = tmp_path / 'runtime.vbs'
|
|
script.write_text('fixture runtime')
|
|
root = tmp_path / 'data'
|
|
preparer = QuickRunPreparer(root, script_source=script, run_token_factory=lambda: 'first',
|
|
cscript_resolver=lambda: Path('cscript.exe'))
|
|
projects = (SimpleNamespace(path=project, selected=True, available=True),)
|
|
rules = (SimpleNamespace(old_part_number='OLD', new_part_number='NEW', enabled=True),)
|
|
handoff = preparer.prepare('task', projects, rules, copy_part_number_to_value=True)
|
|
job = _read_commands(handoff.command_path)[0][1]
|
|
snapshot = handoff.command_path.parent / f'snapshot-{job}.tsv'
|
|
snapshot.write_text(f'M\t{project}\tOLD\tNEW\tA:1\tpart_label\nC\tuid\tR1\t0\t0\t0\t1\tpins\n', encoding='utf-8')
|
|
metadata, components = read_snapshot(snapshot)
|
|
report = {'stage': 'native_replace_verified', 'transaction': transaction_binding(
|
|
snapshot, metadata, components, ['library', 'NEW', 'symbol', '', 'cap', 'label'], 'symbol')}
|
|
snapshot.with_suffix('.tsv.native.json').write_text(json.dumps(report), encoding='utf-8')
|
|
handoff.status_path.write_text('job_id\tstate\tcode\tmessage\n' + job + '\tawaiting_native_replace\t\tstarted\n', encoding='utf-8')
|
|
lock = handoff.command_path.parent / 'uncertain.flag'
|
|
lock.write_text('old timeout')
|
|
return SimpleNamespace(preparer=preparer, root=root, projects=projects, rules=rules, snapshot=snapshot,
|
|
job=job, handoff=handoff, lock=lock, report=report)
|
|
|
|
|
|
def invoke(fixture, **kwargs):
|
|
return recover(fixture.preparer, 'task', fixture.projects, fixture.rules,
|
|
copy_part_number_to_value=True, confirmed=True, **kwargs)
|
|
|
|
|
|
def worker_result(fixture, state='restored_ok', job=None, code=0):
|
|
def run(command, timeout):
|
|
worker = Path(command[3])
|
|
assert worker != fixture.lock.parent
|
|
assert (worker / 'recovery.request').read_text() == fixture.job
|
|
assert timeout == 150
|
|
assert fixture.lock.exists()
|
|
(worker / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n' + (job or fixture.job) + '\t' + state + '\t\tverified\n')
|
|
return subprocess.CompletedProcess(command, code, '', '')
|
|
return run
|
|
|
|
|
|
@pytest.mark.parametrize('hard_failure', [False, True])
|
|
def test_legacy_interruption_without_flag_can_be_reconciled_only_with_bound_evidence(interrupted, hard_failure):
|
|
interrupted.lock.unlink()
|
|
if hard_failure:
|
|
with interrupted.handoff.status_path.open('a', encoding='utf-8') as stream:
|
|
stream.write(interrupted.job + '\thard_failure\t1\tnative write failed\n')
|
|
assert interrupted.preparer.has_interrupted('task')
|
|
assert not interrupted.preparer.has_interrupted('other-task')
|
|
interrupted.preparer._process_runner = worker_result(interrupted, 'retry_ready')
|
|
result = invoke(interrupted)
|
|
assert result.state == 'retry_ready'
|
|
assert not interrupted.lock.exists()
|
|
assert not interrupted.preparer.has_interrupted('task')
|
|
|
|
|
|
def test_preflight_failure_without_transaction_does_not_offer_unsafe_recovery(interrupted):
|
|
interrupted.lock.unlink()
|
|
with interrupted.handoff.status_path.open('a', encoding='utf-8') as stream:
|
|
stream.write(interrupted.job + '\thard_failure\t1\tmissing target\n')
|
|
interrupted.snapshot.with_suffix('.tsv.native.json').write_text('{"stage":"preflight","ok":false}', encoding='utf-8')
|
|
assert not interrupted.preparer.has_interrupted('task')
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert not interrupted.lock.exists()
|
|
|
|
|
|
def test_recovery_requires_explicit_confirmation(interrupted):
|
|
with pytest.raises(TaskCenterError, match='quick_run_recovery_confirmation'):
|
|
recover(interrupted.preparer, 'task', interrupted.projects, interrupted.rules)
|
|
assert interrupted.lock.exists()
|
|
|
|
|
|
def test_verified_recovery_preserves_evidence_and_commits_once(interrupted):
|
|
interrupted.preparer._process_runner = worker_result(interrupted)
|
|
snapshot = interrupted.snapshot.read_bytes()
|
|
result = invoke(interrupted)
|
|
assert result.state == 'restored_ok'
|
|
assert result.next_project_path is None
|
|
assert not interrupted.lock.exists()
|
|
assert interrupted.snapshot.read_bytes() == snapshot
|
|
assert _read_status_details(interrupted.handoff.status_path)[-1][1] == 'restored_ok'
|
|
audit = next((interrupted.root / 'quick-run-history').glob('recovery-*'))
|
|
assert (audit / 'resolved-uncertain.flag').read_text() == 'old timeout'
|
|
assert (audit / 'evidence-before' / interrupted.snapshot.name).read_bytes() == snapshot
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
|
|
|
|
def test_r12_part_number_queue_keeps_recorded_policy_during_recovery(interrupted):
|
|
command_path = interrupted.handoff.command_path
|
|
lines = [line.split('\t') for line in command_path.read_text(encoding='utf-8').splitlines()]
|
|
for column in ('value_mode', 'base_value_mode'):
|
|
lines[1][lines[0].index(column)] = 'part_number'
|
|
command_path.write_text('\n'.join('\t'.join(line) for line in lines) + '\n', encoding='utf-8')
|
|
interrupted.snapshot.write_text(
|
|
interrupted.snapshot.read_text(encoding='utf-8').replace('\tpart_label\n', '\tpart_number\n'),
|
|
encoding='utf-8',
|
|
)
|
|
metadata, components = read_snapshot(interrupted.snapshot)
|
|
report = {'stage': 'native_replace_verified', 'transaction': transaction_binding(
|
|
interrupted.snapshot, metadata, components,
|
|
['library', 'NEW', 'symbol', '', 'cap', 'label'], 'symbol')}
|
|
interrupted.snapshot.with_suffix('.tsv.native.json').write_text(json.dumps(report), encoding='utf-8')
|
|
|
|
def worker(command, timeout):
|
|
worker_root = Path(command[3])
|
|
assert '\tpart_number\t' in (worker_root / 'command.tsv').read_text(encoding='utf-8')
|
|
return worker_result(interrupted)(command, timeout)
|
|
|
|
interrupted.preparer._process_runner = worker
|
|
result = invoke(interrupted)
|
|
assert result.state == 'restored_ok'
|
|
assert result.next_project_path is None
|
|
assert interrupted.preparer.progress(
|
|
'task', interrupted.projects, interrupted.rules, copy_part_number_to_value=True
|
|
).queue_complete
|
|
|
|
|
|
def test_page_addressed_recovery_keeps_the_original_sheet_binding(interrupted):
|
|
project = interrupted.projects[0].path
|
|
interrupted.handoff.command_path.write_text(
|
|
'task_id\tjob_id\tproject_path\tschematic_name\tsheet_number\told_part\tnew_part\tvalue_mode\n'
|
|
f'task\t{interrupted.job}\t{project}\tA\t1\tOLD\tNEW\tpart_label\n',
|
|
encoding='utf-8',
|
|
)
|
|
(interrupted.handoff.command_path.parent / 'sheets.tsv').write_text(
|
|
'project_path\tschematic_name\tsheet_number\n'
|
|
f'{project}\tA\t1\n',
|
|
encoding='utf-8',
|
|
)
|
|
|
|
def worker(command, timeout):
|
|
worker_root = Path(command[3])
|
|
header = (worker_root / 'command.tsv').read_text(encoding='utf-8').splitlines()[0].split('\t')
|
|
row = (worker_root / 'command.tsv').read_text(encoding='utf-8').splitlines()[1].split('\t')
|
|
assert row[header.index('schematic_name')] == 'A'
|
|
assert row[header.index('sheet_number')] == '1'
|
|
assert (worker_root / 'sheets.tsv').is_file()
|
|
(worker_root / 'status.tsv').write_text(
|
|
f'job_id\tstate\tcode\tmessage\n{interrupted.job}\trestored_ok\t\tverified\n',
|
|
encoding='utf-8',
|
|
)
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
|
|
interrupted.preparer._process_runner = worker
|
|
result = invoke(interrupted)
|
|
assert result.state == 'restored_ok'
|
|
assert result.next_project_path is None
|
|
|
|
|
|
def test_unsaved_recovery_creates_fresh_job_without_replacing(interrupted):
|
|
interrupted.preparer._process_runner = worker_result(interrupted, 'retry_ready')
|
|
result = invoke(interrupted)
|
|
assert result.state == 'retry_ready'
|
|
assert result.next_project_path == interrupted.projects[0].path
|
|
assert _read_commands(interrupted.handoff.command_path)[0][1] != interrupted.job
|
|
assert _read_status_details(interrupted.handoff.status_path) == ()
|
|
assert interrupted.snapshot.exists()
|
|
|
|
|
|
@pytest.mark.parametrize(('state', 'job', 'code'), [('hard_failure', None, 1), ('restored_ok', 'foreign', 0),
|
|
('awaiting_native_replace', None, 0), ('restored_ok', None, 2)])
|
|
def test_uncorrelated_or_failed_recovery_never_unlocks(interrupted, state, job, code):
|
|
before = interrupted.handoff.status_path.read_bytes()
|
|
interrupted.preparer._process_runner = worker_result(interrupted, state, job, code)
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.exists()
|
|
assert interrupted.handoff.status_path.read_bytes() == before
|
|
|
|
|
|
def test_recovery_timeout_keeps_old_lock(interrupted):
|
|
def timeout(command, seconds):
|
|
raise subprocess.TimeoutExpired(command, seconds)
|
|
interrupted.preparer._process_runner = timeout
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.read_text() == 'old timeout'
|
|
|
|
|
|
def test_snapshot_binding_mismatch_prevents_worker_start(interrupted):
|
|
interrupted.snapshot.write_text(interrupted.snapshot.read_text() + '\n')
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.exists()
|
|
assert not (interrupted.root / 'quick-run-history').exists()
|
|
|
|
|
|
def test_concurrent_evidence_change_does_not_commit(interrupted):
|
|
worker = worker_result(interrupted)
|
|
def change(command, seconds):
|
|
result = worker(command, seconds)
|
|
interrupted.lock.write_text('new protection')
|
|
return result
|
|
interrupted.preparer._process_runner = change
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.read_text() == 'new protection'
|
|
|
|
|
|
def test_other_pending_job_prevents_global_unlock(interrupted):
|
|
with interrupted.handoff.status_path.open('a') as stream:
|
|
stream.write('other-job\thard_failure\t\tunknown\n')
|
|
with pytest.raises(TaskCenterError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.exists()
|
|
|
|
|
|
def test_failed_commit_restores_status_and_lock(interrupted, monkeypatch):
|
|
from dx_productivity.trial_task_center import recovery
|
|
interrupted.preparer._process_runner = worker_result(interrupted)
|
|
original = interrupted.handoff.status_path.read_bytes()
|
|
real_replace = recovery.os.replace
|
|
def fail_unlock(source, target):
|
|
if Path(source) == interrupted.lock:
|
|
raise PermissionError('locked')
|
|
return real_replace(source, target)
|
|
monkeypatch.setattr(recovery.os, 'replace', fail_unlock)
|
|
with pytest.raises(PermissionError):
|
|
invoke(interrupted)
|
|
assert interrupted.lock.exists()
|
|
assert interrupted.handoff.status_path.read_bytes() == original
|
|
|
|
|
|
@pytest.mark.parametrize('initial', ['ready', 'paused', 'failed'])
|
|
def test_service_recovery_updates_queue_and_database(interrupted, initial):
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.models import TaskStatus
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
|
|
store = TaskStore(interrupted.root / 'task-center.db')
|
|
service = TaskCenterService(store, configure_logging(interrupted.root), quick_run_preparer=interrupted.preparer)
|
|
task = service.create_task('Interrupted')
|
|
service.set_copy_part_number_to_value(task.task_id, True)
|
|
projects = service.scan_projects(task.task_id, interrupted.projects[0].path.parent)
|
|
service.set_project_selected(projects[0].project_id, True)
|
|
service.add_rule(task.task_id, 'OLD', 'NEW')
|
|
command_path = interrupted.handoff.command_path
|
|
command_path.write_text(command_path.read_text().replace('task\t', task.task_id + '\t'), encoding='utf-8')
|
|
if store.get_task(task.task_id).status is TaskStatus.DRAFT:
|
|
store.set_task_status(task.task_id, TaskStatus.READY)
|
|
if initial == 'paused':
|
|
store.set_task_status(task.task_id, TaskStatus.RUNNING)
|
|
store.set_task_status(task.task_id, TaskStatus.PAUSED)
|
|
elif initial == 'failed':
|
|
store.set_task_status(task.task_id, TaskStatus.FAILED)
|
|
interrupted.preparer._process_runner = worker_result(interrupted)
|
|
result = service.recover_quick_run(task.task_id, confirmed=True)
|
|
assert result.state == 'restored_ok'
|
|
assert store.get_task(task.task_id).status is TaskStatus.COMPLETED
|
|
assert service.quick_run_progress(task.task_id).queue_complete
|
|
assert not service.has_interrupted_quick_run(task.task_id)
|