271 lines
13 KiB
Python
271 lines
13 KiB
Python
import json
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.dsh.run_journal import RunJournal
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.models import TaskStatus
|
|
from dx_productivity.trial_task_center.task_queue import _plan_hash, TaskQueueResult, TaskQueueEntry
|
|
|
|
|
|
@pytest.fixture
|
|
def project_service(tmp_path):
|
|
root = tmp_path / 'projects'
|
|
for name in ('A', 'B', 'C'):
|
|
path = root / name / 'board.prj'
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text('KEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n')
|
|
data = tmp_path / 'data'
|
|
preparer = QuickRunPreparer(data, sheet_discoverer=lambda _path: (('MAIN', '1'),))
|
|
store = TaskStore(tmp_path / 'tasks.db')
|
|
service = TaskCenterService(store, configure_logging(data), quick_run_preparer=preparer)
|
|
tasks = service.create_project_tasks(root)
|
|
service.add_rule(tasks[0].task_id, 'OLD', 'NEW')
|
|
ids = [task.task_id for task in tasks]
|
|
hashes = {key: _plan_hash(service.for_project_task(key), key) for key in ids}
|
|
yield service, store, data, ids, hashes
|
|
store.close()
|
|
|
|
|
|
def evidence(service, task_id, state='no_match', code='batch_closed'):
|
|
context = service.task_context(task_id)
|
|
root = service.for_project_task(task_id)._quick_run_preparer.data_root / 'quick-run'
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\tvalue_mode\tcustom_value\tsync_nc\tbase_value_mode\n'
|
|
f'{task_id}\tjob\t{context.project.path}\tOLD\tNEW\tpreserve\t\t1\tpreserve\n', encoding='utf-8')
|
|
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n'
|
|
f'job\t{state}\t{code}\tfixture evidence\n', encoding='utf-8')
|
|
|
|
|
|
def checkpoint(data, run_id, ids, hashes, states=None):
|
|
path = data / 'task-queues' / f'{run_id}.json'
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
|
|
'entries': {key: {'state': (states or {}).get(key, 'pending'), 'message': ''} for key in ids}}))
|
|
|
|
|
|
def test_begin_is_durable_reserves_original_queue_and_rejects_rebinding(project_service):
|
|
service, store, data, ids, hashes = project_service
|
|
row = RunJournal(data).begin('a' * 32, ids, hashes)
|
|
assert row['queue_id'] == 'a' * 32 and row['task_ids'] == ids and row['status'] == 'running'
|
|
reopened = RunJournal(data)
|
|
assert reopened.get('a' * 32) == row
|
|
assert reopened.list_runs()[0] == row
|
|
assert reopened.begin('a' * 32, ids, hashes) == row
|
|
with pytest.raises(TaskCenterError):
|
|
reopened.begin('a' * 32, ids[::-1], hashes)
|
|
|
|
|
|
def test_finish_preserves_receipt_across_restart(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'completed'),), True, ids[1])
|
|
journal.finish('a' * 32, result)
|
|
row = RunJournal(data).get('a' * 32)
|
|
assert row['status'] == 'paused' and row['result'] == json.loads(json.dumps(asdict(result)))
|
|
assert row['finished_utc']
|
|
|
|
|
|
def test_failed_queue_receipt_does_not_claim_success(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'failed'),), False)
|
|
assert journal.finish('a' * 32, result)['status'] == 'failed'
|
|
|
|
|
|
def test_inspect_accepts_frozen_workspace_checkpoint(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('b' * 32, ids, hashes)
|
|
path = data / 'workspace-queues' / ('b' * 32 + '.json')
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
|
|
'phase': 'primary', 'entries': {key: {'state': 'pending', 'message': '',
|
|
'released': False, 'problems_done': False, 'checkpoint_id': str(index) * 32}
|
|
for index, key in enumerate(ids, 1)}}))
|
|
result = journal.inspect(service, 'b' * 32)
|
|
assert result['queue_available'] and result['resume_ready']
|
|
assert result['remaining_task_ids'] == ids
|
|
|
|
|
|
@pytest.mark.parametrize('interrupted', [False, True])
|
|
def test_shared_safe_primary_failure_can_hold_remaining_commands_on_resume(project_service, monkeypatch, interrupted):
|
|
from dataclasses import replace
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('c' * 32, ids, hashes)
|
|
evidence(service, ids[0], state='batch_failed')
|
|
service._record_safe_queue_failure(ids[0], 'failed rule; other sheet held')
|
|
checkpoint(data, 'c' * 32, ids, hashes, {ids[0]: 'failed'})
|
|
path = data / 'task-queues' / ('c' * 32 + '.json')
|
|
saved = json.loads(path.read_text())
|
|
saved['phase'] = 'primary'
|
|
path.write_text(json.dumps(saved))
|
|
original = service.task_context
|
|
def context(task_id):
|
|
value = original(task_id)
|
|
return (replace(value, progress=replace(value.progress, queue_complete=False),
|
|
interrupted=interrupted) if task_id == ids[0] else value)
|
|
monkeypatch.setattr(service, 'task_context', context)
|
|
result = journal.inspect(service, 'c' * 32)
|
|
if interrupted:
|
|
assert not result['resume_ready'] and 'interrupted' in result['blockers']
|
|
else:
|
|
assert result['resume_ready'], result['blockers']
|
|
assert ids[0] in result['released_primary_task_ids']
|
|
assert result['remaining_task_ids'] == ids
|
|
|
|
|
|
@pytest.mark.parametrize('isolated', [True, False])
|
|
def test_released_isolated_project_does_not_block_resume_of_other_projects(project_service, isolated):
|
|
from dx_productivity.dsh.run_journal import resume_preflight
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('d' * 32, ids, hashes)
|
|
evidence(service, ids[0], state='batch_saved_unverified' if isolated else 'batch_failed')
|
|
entries = {key: {'state': ('blocked' if isolated else 'failed') if index == 0 else 'pending', 'message': 'problem' if index == 0 else '',
|
|
'released': index == 0, 'problems_done': False, 'checkpoint_id': str(index + 1) * 32}
|
|
for index, key in enumerate(ids)}
|
|
entries[ids[0]]['supervisor_batch_id'] = 'e' * 32
|
|
path = data / 'workspace-queues' / ('d' * 32 + '.json')
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes,
|
|
'phase': 'primary' if isolated else 'problems', 'entries': entries}))
|
|
result = journal.inspect(service, 'd' * 32)
|
|
assert result['queue_available'] and result['resume_ready'], result['blockers']
|
|
assert result['remaining_task_ids'] == (ids[1:] if isolated else ids)
|
|
assert result['isolated_task_ids'] == ([ids[0]] if isolated else [])
|
|
assert result['tasks'][0]['interrupted'] == isolated
|
|
report = resume_preflight(service, result)
|
|
assert report['ready']
|
|
assert report['tasks'][0]['isolated'] == isolated
|
|
|
|
|
|
def test_unpaused_blocked_project_is_not_a_completed_run(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('b' * 32, ids, hashes)
|
|
result = TaskQueueResult('b' * 32, (TaskQueueEntry(ids[0], 'blocked'),), False)
|
|
assert journal.finish('b' * 32, result)['status'] == 'blocked'
|
|
|
|
|
|
def test_confirmed_run_reports_isolated_blocked_project_as_issue(project_service, monkeypatch):
|
|
from threading import Event
|
|
from dx_productivity.dsh.execution_runs import execute_confirmed_run
|
|
from dx_productivity.dsh.task_lifecycle import preflight
|
|
service, _, data, ids, _ = project_service
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *_args, **_kwargs:
|
|
TaskQueueResult('c' * 32, tuple(TaskQueueEntry(task_id, 'blocked' if index == 0 else 'completed')
|
|
for index, task_id in enumerate(ids)), False))
|
|
result = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), 'c' * 32)
|
|
assert result['status'] == 'blocked'
|
|
|
|
|
|
@pytest.mark.parametrize('status,verified,expected', [
|
|
('verified', True, 'completed'), ('different', False, 'failed'),
|
|
('unavailable', False, 'failed'), ('verified', False, 'failed'),
|
|
])
|
|
def test_netlist_acceptance_controls_run_result_without_replaying_saved_entries(
|
|
project_service, monkeypatch, status, verified, expected):
|
|
from threading import Event
|
|
from dx_productivity.dsh.execution_runs import execute_confirmed_run
|
|
from dx_productivity.dsh.task_lifecycle import preflight
|
|
service, _, data, ids, _ = project_service
|
|
ids = ids[:1]
|
|
report = {ids[0]: {'status': status, 'verified': verified, 'report_path': 'verification.json'}}
|
|
monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *_a, **_kw:
|
|
TaskQueueResult('c' * 32, (TaskQueueEntry(ids[0], 'completed'),), False, netlist_verification=report))
|
|
result = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), 'c' * 32)
|
|
assert result['status'] == expected
|
|
assert result['execution_status'] == 'completed'
|
|
assert result['entries'][0]['state'] == 'completed'
|
|
assert RunJournal(data).get('c' * 32)['status'] == expected
|
|
|
|
|
|
def test_native_confirmed_resume_marks_running_and_retains_last_receipt(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
result = TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'paused'),), True, ids[0])
|
|
prior = journal.finish('a' * 32, result)
|
|
row = journal.resume('a' * 32)
|
|
assert row['status'] == 'running' and row['finished_utc'] is None
|
|
assert row['result'] == prior['result'] and row['queue_id'] == 'a' * 32
|
|
journal.finish('a' * 32, TaskQueueResult('a' * 32, (TaskQueueEntry(ids[0], 'completed'),), False))
|
|
with pytest.raises(TaskCenterError):
|
|
journal.resume('a' * 32)
|
|
|
|
|
|
def test_inspect_reports_invalid_checkpoint_without_trusting_partial_content(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
path = data / 'task-queues' / ('a' * 32 + '.json')
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(json.dumps({'version': 1, 'task_ids': ids[::-1]}))
|
|
result = journal.inspect(service, 'a' * 32)
|
|
assert not result['queue_available'] and not result['resume_ready']
|
|
assert result['blockers'] == ['queue_invalid']
|
|
|
|
|
|
def test_inspect_reads_real_counters_and_skips_completed_original_scope(project_service):
|
|
service, store, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
checkpoint(data, 'a' * 32, ids, hashes, {ids[0]: 'completed'})
|
|
evidence(service, ids[0])
|
|
store.set_task_status(ids[0], TaskStatus.READY)
|
|
store.set_task_status(ids[0], TaskStatus.RUNNING)
|
|
store.set_task_status(ids[0], TaskStatus.COMPLETED)
|
|
result = RunJournal(data).inspect(service, 'a' * 32)
|
|
assert result['queue_available'] and result['resume_ready']
|
|
assert result['task_ids'] == ids and result['remaining_task_ids'] == ids[1:]
|
|
assert result['counters']['no_match_items'] == 1
|
|
assert result['tasks'][0]['no_match_items'] == 1
|
|
assert not result['cad_execution_started']
|
|
|
|
|
|
def test_inspect_blocks_missing_checkpoint_stale_plan_and_uncertainty(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
result = journal.inspect(service, 'a' * 32)
|
|
assert not result['queue_available'] and result['resume_ready']
|
|
evidence(service, ids[0])
|
|
assert 'queue_missing' in journal.inspect(service, 'a' * 32)['blockers']
|
|
evidence_root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
|
|
(evidence_root / 'command.tsv').unlink()
|
|
(evidence_root / 'status.tsv').unlink()
|
|
checkpoint(data, 'a' * 32, ids, hashes)
|
|
rule = service.rules(ids[1])[0]
|
|
service.update_rule(rule.rule_id, old_part_number='OLD', new_part_number='CHANGED', enabled=True)
|
|
evidence(service, ids[0], 'batch_saved_unverified', '')
|
|
result = journal.inspect(service, 'a' * 32)
|
|
assert not result['resume_ready']
|
|
assert 'plan_changed' in result['blockers'] and 'interrupted' in result['blockers']
|
|
|
|
|
|
def test_inspect_blocks_archived_unfinished_tasks(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
journal = RunJournal(data)
|
|
journal.begin('a' * 32, ids, hashes)
|
|
checkpoint(data, 'a' * 32, ids, hashes)
|
|
service.delete_task(ids[0])
|
|
result = journal.inspect(service, 'a' * 32)
|
|
assert not result['resume_ready'] and 'preflight_blocked' in result['blockers']
|
|
|
|
|
|
@pytest.mark.parametrize('run_id,ids,hashes', [('../outside', ['a'], {'a': 'b' * 64}),
|
|
('a' * 32, ['a', 'a'], {'a': 'b' * 64}), ('a' * 32, ['a'], {'other': 'b' * 64})])
|
|
def test_begin_rejects_unsafe_or_inconsistent_scope(tmp_path, run_id, ids, hashes):
|
|
with pytest.raises(TaskCenterError):
|
|
RunJournal(tmp_path).begin(run_id, ids, hashes)
|