Files
dxdesigner-part-replacement/tests/dsh/test_run_journal.py

271 lines
13 KiB
Python

import json
from dataclasses import asdict
from pathlib import Path
import pytest
from dx_productivity.dsh.run_journal import RunJournal
from dx_productivity.trial_task_center.errors import TaskCenterError
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.models import TaskStatus
from dx_productivity.trial_task_center.task_queue import _plan_hash, TaskQueueResult, TaskQueueEntry
@pytest.fixture
def project_service(tmp_path):
root = tmp_path / 'projects'
for name in ('A', 'B', 'C'):
path = root / name / 'board.prj'
path.parent.mkdir(parents=True)
path.write_text('KEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n')
data = tmp_path / 'data'
preparer = QuickRunPreparer(data, sheet_discoverer=lambda _path: (('MAIN', '1'),))
store = TaskStore(tmp_path / 'tasks.db')
service = TaskCenterService(store, configure_logging(data), quick_run_preparer=preparer)
tasks = service.create_project_tasks(root)
service.add_rule(tasks[0].task_id, 'OLD', 'NEW')
ids = [task.task_id for task in tasks]
hashes = {key: _plan_hash(service.for_project_task(key), key) for key in ids}
yield service, store, data, ids, hashes
store.close()
def evidence(service, task_id, state='no_match', code='batch_closed'):
context = service.task_context(task_id)
root = service.for_project_task(task_id)._quick_run_preparer.data_root / 'quick-run'
root.mkdir(parents=True, exist_ok=True)
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\tvalue_mode\tcustom_value\tsync_nc\tbase_value_mode\n'
f'{task_id}\tjob\t{context.project.path}\tOLD\tNEW\tpreserve\t\t1\tpreserve\n', encoding='utf-8')
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n'
f'job\t{state}\t{code}\tfixture evidence\n', encoding='utf-8')
def checkpoint(data, run_id, ids, hashes, states=None):
path = data / 'task-queues' / f'{run_id}.json'
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
'entries': {key: {'state': (states or {}).get(key, 'pending'), 'message': ''} for key in ids}}))
def test_begin_is_durable_reserves_original_queue_and_rejects_rebinding(project_service):
service, store, data, ids, hashes = project_service
row = RunJournal(data).begin('a' * 32, ids, hashes)
assert row['queue_id'] == 'a' * 32 and row['task_ids'] == ids and row['status'] == 'running'
reopened = RunJournal(data)
assert reopened.get('a' * 32) == row
assert reopened.list_runs()[0] == row
assert reopened.begin('a' * 32, ids, hashes) == row
with pytest.raises(TaskCenterError):
reopened.begin('a' * 32, ids[::-1], hashes)
def test_finish_preserves_receipt_across_restart(project_service):
_, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'completed'),), True, ids[1])
journal.finish('a' * 32, result)
row = RunJournal(data).get('a' * 32)
assert row['status'] == 'paused' and row['result'] == json.loads(json.dumps(asdict(result)))
assert row['finished_utc']
def test_failed_queue_receipt_does_not_claim_success(project_service):
_, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'failed'),), False)
assert journal.finish('a' * 32, result)['status'] == 'failed'
def test_inspect_accepts_frozen_workspace_checkpoint(project_service):
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('b' * 32, ids, hashes)
path = data / 'workspace-queues' / ('b' * 32 + '.json')
path.parent.mkdir(parents=True)
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
'phase': 'primary', 'entries': {key: {'state': 'pending', 'message': '',
'released': False, 'problems_done': False, 'checkpoint_id': str(index) * 32}
for index, key in enumerate(ids, 1)}}))
result = journal.inspect(service, 'b' * 32)
assert result['queue_available'] and result['resume_ready']
assert result['remaining_task_ids'] == ids
@pytest.mark.parametrize('interrupted', [False, True])
def test_shared_safe_primary_failure_can_hold_remaining_commands_on_resume(project_service, monkeypatch, interrupted):
from dataclasses import replace
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('c' * 32, ids, hashes)
evidence(service, ids[0], state='batch_failed')
service._record_safe_queue_failure(ids[0], 'failed rule; other sheet held')
checkpoint(data, 'c' * 32, ids, hashes, {ids[0]: 'failed'})
path = data / 'task-queues' / ('c' * 32 + '.json')
saved = json.loads(path.read_text())
saved['phase'] = 'primary'
path.write_text(json.dumps(saved))
original = service.task_context
def context(task_id):
value = original(task_id)
return (replace(value, progress=replace(value.progress, queue_complete=False),
interrupted=interrupted) if task_id == ids[0] else value)
monkeypatch.setattr(service, 'task_context', context)
result = journal.inspect(service, 'c' * 32)
if interrupted:
assert not result['resume_ready'] and 'interrupted' in result['blockers']
else:
assert result['resume_ready'], result['blockers']
assert ids[0] in result['released_primary_task_ids']
assert result['remaining_task_ids'] == ids
@pytest.mark.parametrize('isolated', [True, False])
def test_released_isolated_project_does_not_block_resume_of_other_projects(project_service, isolated):
from dx_productivity.dsh.run_journal import resume_preflight
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('d' * 32, ids, hashes)
evidence(service, ids[0], state='batch_saved_unverified' if isolated else 'batch_failed')
entries = {key: {'state': ('blocked' if isolated else 'failed') if index == 0 else 'pending', 'message': 'problem' if index == 0 else '',
'released': index == 0, 'problems_done': False, 'checkpoint_id': str(index + 1) * 32}
for index, key in enumerate(ids)}
entries[ids[0]]['supervisor_batch_id'] = 'e' * 32
path = data / 'workspace-queues' / ('d' * 32 + '.json')
path.parent.mkdir(parents=True)
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
'phase': 'primary' if isolated else 'problems', 'entries': entries}))
result = journal.inspect(service, 'd' * 32)
assert result['queue_available'] and result['resume_ready'], result['blockers']
assert result['remaining_task_ids'] == (ids[1:] if isolated else ids)
assert result['isolated_task_ids'] == ([ids[0]] if isolated else [])
assert result['tasks'][0]['interrupted'] == isolated
report = resume_preflight(service, result)
assert report['ready']
assert report['tasks'][0]['isolated'] == isolated
def test_unpaused_blocked_project_is_not_a_completed_run(project_service):
_, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('b' * 32, ids, hashes)
result = TaskQueueResult('b' * 32, (TaskQueueEntry(ids[0], 'blocked'),), False)
assert journal.finish('b' * 32, result)['status'] == 'blocked'
def test_confirmed_run_reports_isolated_blocked_project_as_issue(project_service, monkeypatch):
from threading import Event
from dx_productivity.dsh.execution_runs import execute_confirmed_run
from dx_productivity.dsh.task_lifecycle import preflight
service, _, data, ids, _ = project_service
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *_args, **_kwargs:
TaskQueueResult('c' * 32, tuple(TaskQueueEntry(task_id, 'blocked' if index == 0 else 'completed')
for index, task_id in enumerate(ids)), False))
result = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), 'c' * 32)
assert result['status'] == 'blocked'
@pytest.mark.parametrize('status,verified,expected', [
('verified', True, 'completed'), ('different', False, 'failed'),
('unavailable', False, 'failed'), ('verified', False, 'failed'),
])
def test_netlist_acceptance_controls_run_result_without_replaying_saved_entries(
project_service, monkeypatch, status, verified, expected):
from threading import Event
from dx_productivity.dsh.execution_runs import execute_confirmed_run
from dx_productivity.dsh.task_lifecycle import preflight
service, _, data, ids, _ = project_service
ids = ids[:1]
report = {ids[0]: {'status': status, 'verified': verified, 'report_path': 'verification.json'}}
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *_a, **_kw:
TaskQueueResult('c' * 32, (TaskQueueEntry(ids[0], 'completed'),), False, netlist_verification=report))
result = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), 'c' * 32)
assert result['status'] == expected
assert result['execution_status'] == 'completed'
assert result['entries'][0]['state'] == 'completed'
assert RunJournal(data).get('c' * 32)['status'] == expected
def test_native_confirmed_resume_marks_running_and_retains_last_receipt(project_service):
_, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'paused'),), True, ids[0])
prior = journal.finish('a' * 32, result)
row = journal.resume('a' * 32)
assert row['status'] == 'running' and row['finished_utc'] is None
assert row['result'] == prior['result'] and row['queue_id'] == 'a' * 32
journal.finish('a' * 32, TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'completed'),), False))
with pytest.raises(TaskCenterError):
journal.resume('a' * 32)
def test_inspect_reports_invalid_checkpoint_without_trusting_partial_content(project_service):
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
path = data / 'task-queues' / ('a' * 32 + '.json')
path.parent.mkdir(parents=True)
path.write_text(json.dumps({'version': 1, 'task_ids': ids[::-1]}))
result = journal.inspect(service, 'a' * 32)
assert not result['queue_available'] and not result['resume_ready']
assert result['blockers'] == ['queue_invalid']
def test_inspect_reads_real_counters_and_skips_completed_original_scope(project_service):
service, store, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
checkpoint(data, 'a' * 32, ids, hashes, {ids[0]: 'completed'})
evidence(service, ids[0])
store.set_task_status(ids[0], TaskStatus.READY)
store.set_task_status(ids[0], TaskStatus.RUNNING)
store.set_task_status(ids[0], TaskStatus.COMPLETED)
result = RunJournal(data).inspect(service, 'a' * 32)
assert result['queue_available'] and result['resume_ready']
assert result['task_ids'] == ids and result['remaining_task_ids'] == ids[1:]
assert result['counters']['no_match_items'] == 1
assert result['tasks'][0]['no_match_items'] == 1
assert not result['cad_execution_started']
def test_inspect_blocks_missing_checkpoint_stale_plan_and_uncertainty(project_service):
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
result = journal.inspect(service, 'a' * 32)
assert not result['queue_available'] and result['resume_ready']
evidence(service, ids[0])
assert 'queue_missing' in journal.inspect(service, 'a' * 32)['blockers']
evidence_root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
(evidence_root / 'command.tsv').unlink()
(evidence_root / 'status.tsv').unlink()
checkpoint(data, 'a' * 32, ids, hashes)
rule = service.rules(ids[1])[0]
service.update_rule(rule.rule_id, old_part_number='OLD', new_part_number='CHANGED', enabled=True)
evidence(service, ids[0], 'batch_saved_unverified', '')
result = journal.inspect(service, 'a' * 32)
assert not result['resume_ready']
assert 'plan_changed' in result['blockers'] and 'interrupted' in result['blockers']
def test_inspect_blocks_archived_unfinished_tasks(project_service):
service, _, data, ids, hashes = project_service
journal = RunJournal(data)
journal.begin('a' * 32, ids, hashes)
checkpoint(data, 'a' * 32, ids, hashes)
service.delete_task(ids[0])
result = journal.inspect(service, 'a' * 32)
assert not result['resume_ready'] and 'preflight_blocked' in result['blockers']
@pytest.mark.parametrize('run_id,ids,hashes', [('../outside', ['a'], {'a': 'b' * 64}),
('a' * 32, ['a', 'a'], {'a': 'b' * 64}), ('a' * 32, ['a'], {'other': 'b' * 64})])
def test_begin_rejects_unsafe_or_inconsistent_scope(tmp_path, run_id, ids, hashes):
with pytest.raises(TaskCenterError):
RunJournal(tmp_path).begin(run_id, ids, hashes)