284 lines
14 KiB
Python
284 lines
14 KiB
Python
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.dsh.scheduling import ScheduleStore
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
from dx_productivity.trial_task_center.models import TaskStatus
|
|
from test_run_journal import project_service, evidence
|
|
|
|
|
|
NOW = datetime(2026, 10, 30, 12, tzinfo=timezone.utc)
|
|
|
|
|
|
def spec(ids, **extra):
|
|
return {'task_ids': ids, 'priority': 'normal', 'not_before': '2026-10-30T20:00:00+08:00', **extra}
|
|
|
|
|
|
def test_save_normalizes_utc_and_preserves_frozen_scope_across_restart(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
row = ScheduleStore(data).save(spec(ids), hashes)
|
|
assert row['not_before'] == '2026-10-30T12:00:00+00:00'
|
|
assert row['status'] == 'pending' and row['auto_start'] is False and row['idle_only'] is True
|
|
assert ScheduleStore(data).get(row['schedule_id']) == row
|
|
|
|
|
|
def test_preview_normalizes_native_draft_without_persisting_or_granting_authority(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
result = schedules.preview(spec(ids, auto_start=True), hashes)
|
|
assert result == {'task_ids': ids, 'priority': 'normal', 'not_before': '2026-10-30T12:00:00+00:00',
|
|
'due_at': None, 'dependencies': [], 'auto_start': True, 'idle_only': True, 'title': ''}
|
|
assert schedules.list_schedules() == []
|
|
|
|
|
|
@pytest.mark.parametrize('extra', [{'confirmed': True}, {'priority': 'highest'}, {'not_before': '2026-10-30T12:00:00'},
|
|
{'auto_start': 'true'}, {'dependencies': ['../bad']}, {'task_ids': []}, {'title': '\x00bad'},
|
|
{'due_at': '2026-10-29T00:00:00Z'}])
|
|
def test_save_rejects_untrusted_or_invalid_specs(project_service, extra):
|
|
_, _, data, ids, hashes = project_service
|
|
with pytest.raises(TaskCenterError):
|
|
ScheduleStore(data).save(spec(ids, **extra), hashes)
|
|
|
|
|
|
def test_due_order_and_manual_confirmation_without_execution(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
normal = schedules.save(spec(ids), hashes)
|
|
urgent = schedules.save(spec(ids, priority='urgent', auto_start=True), hashes)
|
|
rows = schedules.evaluate(service, now=NOW)
|
|
assert [row['schedule_id'] for row in rows] == [urgent['schedule_id'], normal['schedule_id']]
|
|
assert rows[0]['ready'] and rows[0]['requires_confirmation'] is False
|
|
assert rows[0]['dispatchable'] is True # save follows native future-run authorization
|
|
assert rows[0]['dispatch_authority'] == 'native_future_authorization'
|
|
assert rows[1]['requires_confirmation'] and not rows[1]['dispatchable']
|
|
assert not list(data.rglob('command.tsv'))
|
|
assert all('worker_busy' in row['blockers'] for row in schedules.evaluate(service, now=NOW, busy=True))
|
|
|
|
|
|
def test_dispatch_and_result_audit_are_durable_and_automatic_claim_needs_authority(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
manual = schedules.save(spec(ids), hashes)
|
|
with pytest.raises(TaskCenterError, match='schedule_authority'):
|
|
schedules.claim(manual['schedule_id'], automatic=True, run_id='a' * 32)
|
|
assert schedules.get(manual['schedule_id'])['status'] == 'pending'
|
|
assert schedules.events() == []
|
|
row = schedules.save(spec(ids, auto_start=True), hashes)
|
|
schedules.claim(row['schedule_id'], automatic=True, run_id='b' * 32)
|
|
schedules.finish(row['schedule_id'], 'completed', run_id='b' * 32, receipt={'status': 'completed'})
|
|
events = ScheduleStore(data).events()
|
|
assert [e['event'] for e in reversed(events)] == ['dispatch', 'result']
|
|
assert events[1]['payload']['scope'] == {'task_ids': ids, 'plan_hashes': hashes}
|
|
assert events[1]['payload']['authority'] == 'native_future_authorization'
|
|
assert events[0]['payload']['receipt']['status'] == 'completed'
|
|
assert all(e['run_id'] == 'b' * 32 for e in events)
|
|
|
|
|
|
def test_audit_insert_failure_rolls_back_claim_before_execution(project_service):
|
|
import sqlite3
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids, auto_start=True), hashes)
|
|
with sqlite3.connect(schedules.path) as db:
|
|
db.execute("CREATE TRIGGER reject_event BEFORE INSERT ON schedule_events BEGIN SELECT RAISE(ABORT, 'controlled audit fault'); END")
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
schedules.claim(row['schedule_id'], automatic=True, run_id='b' * 32)
|
|
assert ScheduleStore(data).get(row['schedule_id'])['status'] == 'pending'
|
|
|
|
|
|
def test_future_and_changed_plans_do_not_become_ready(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
schedules.save(spec(ids, not_before='2026-10-31T12:00:00Z'), hashes)
|
|
assert 'not_due' in schedules.evaluate(service, now=NOW)[0]['blockers']
|
|
rule = service.rules(ids[0])[0]
|
|
service.update_rule(rule.rule_id, old_part_number='OLD', new_part_number='CHANGED', enabled=True)
|
|
assert 'plan_changed' in schedules.evaluate(service, now=NOW)[0]['blockers']
|
|
|
|
|
|
def test_preflight_blocks_missing_project_and_unreconciled_saved_result(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
schedules.save(spec(ids), hashes)
|
|
evidence(service, ids[0], 'batch_saved_unverified', '')
|
|
row = schedules.evaluate(service, now=NOW)[0]
|
|
assert not row['ready'] and 'preflight_blocked' in row['blockers']
|
|
service.task_context(ids[1]).project.path.unlink()
|
|
assert not schedules.evaluate(service, now=NOW)[0]['ready']
|
|
|
|
|
|
def test_paused_task_requires_explicit_manual_resume(project_service):
|
|
service, store, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
schedules.save(spec(ids, auto_start=True), hashes)
|
|
store.set_task_status(ids[0], TaskStatus.READY)
|
|
store.set_task_status(ids[0], TaskStatus.RUNNING)
|
|
store.set_task_status(ids[0], TaskStatus.PAUSED)
|
|
row = schedules.evaluate(service, now=NOW)[0]
|
|
assert not row['ready'] and 'task_paused' in row['blockers']
|
|
|
|
|
|
def test_multiple_schedules_for_a_task_cannot_hide_dependency_cycle(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
schedules.save(spec(ids[:1], dependencies=ids[1:2]), {ids[0]: hashes[ids[0]]})
|
|
schedules.save(spec(ids[:1], dependencies=[]), {ids[0]: hashes[ids[0]]})
|
|
with pytest.raises(TaskCenterError):
|
|
schedules.save(spec(ids[1:2], dependencies=ids[:1]), {ids[1]: hashes[ids[1]]})
|
|
|
|
|
|
def test_dependencies_require_real_completion_and_verified_evidence(project_service):
|
|
service, store, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
schedules.save(spec(ids[:1], dependencies=ids[1:2]), {ids[0]: hashes[ids[0]]})
|
|
assert 'dependency_incomplete' in schedules.evaluate(service, now=NOW)[0]['blockers']
|
|
store.set_task_status(ids[1], TaskStatus.READY)
|
|
store.set_task_status(ids[1], TaskStatus.RUNNING)
|
|
store.set_task_status(ids[1], TaskStatus.COMPLETED)
|
|
assert 'dependency_incomplete' in schedules.evaluate(service, now=NOW)[0]['blockers']
|
|
evidence(service, ids[1])
|
|
assert schedules.evaluate(service, now=NOW)[0]['ready']
|
|
evidence(service, ids[1], 'batch_saved_unverified', '')
|
|
assert 'dependency_incomplete' in schedules.evaluate(service, now=NOW)[0]['blockers']
|
|
|
|
|
|
def test_self_dependency_and_cross_schedule_cycles_are_rejected(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
with pytest.raises(TaskCenterError):
|
|
schedules.save(spec(ids[:1], dependencies=ids[:1]), {ids[0]: hashes[ids[0]]})
|
|
schedules.save(spec(ids[:1], dependencies=ids[1:2]), {ids[0]: hashes[ids[0]]})
|
|
with pytest.raises(TaskCenterError):
|
|
schedules.save(spec(ids[1:2], dependencies=ids[:1]), {ids[1]: hashes[ids[1]]})
|
|
|
|
|
|
def test_claim_is_exclusive_and_never_replayed_after_restart(project_service):
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids), hashes)
|
|
def claim(_):
|
|
try:
|
|
return ScheduleStore(data).claim(row['schedule_id'])
|
|
except TaskCenterError:
|
|
return None
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
assert sum(value is not None for value in pool.map(claim, range(4))) == 1
|
|
reopened = ScheduleStore(data)
|
|
assert reopened.get(row['schedule_id'])['status'] == 'claimed'
|
|
assert 'already_claimed' in reopened.evaluate(service, now=NOW)[0]['blockers']
|
|
with pytest.raises(TaskCenterError):
|
|
reopened.claim(row['schedule_id'])
|
|
reopened.finish(row['schedule_id'], 'completed', run_id='a' * 32)
|
|
assert reopened.get(row['schedule_id'])['run_id'] == 'a' * 32
|
|
|
|
|
|
def test_cancel_is_durable_and_cannot_cancel_an_owned_claim(project_service):
|
|
_, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids), hashes)
|
|
schedules.cancel(row['schedule_id'])
|
|
assert ScheduleStore(data).get(row['schedule_id'])['status'] == 'cancelled'
|
|
with pytest.raises(TaskCenterError):
|
|
schedules.claim(row['schedule_id'])
|
|
claimed = schedules.save(spec(ids), hashes)
|
|
schedules.claim(claimed['schedule_id'])
|
|
with pytest.raises(TaskCenterError):
|
|
schedules.cancel(claimed['schedule_id'])
|
|
|
|
|
|
@pytest.mark.parametrize('run_state', [None, 'running', 'completed', 'paused', 'failed'])
|
|
def test_reconcile_orphan_claim_preserves_original_run_and_never_replays(project_service, run_state):
|
|
from dx_productivity.dsh.run_journal import RunJournal
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids, auto_start=True), hashes)
|
|
run_id = 'c' * 32
|
|
schedules.claim(row['schedule_id'], run_id=run_id, automatic=True)
|
|
journal = RunJournal(data)
|
|
if run_state:
|
|
journal.begin(run_id, ids, hashes, schedule_id=row['schedule_id'])
|
|
if run_state != 'running':
|
|
journal.finish(run_id, {'status': run_state, 'tasks': [], 'entries': []})
|
|
before = journal.get(run_id)
|
|
reopened = ScheduleStore(data)
|
|
result = reopened.reconcile_claim(service, row['schedule_id'])
|
|
expected = run_state if run_state in {'completed', 'failed'} else 'interrupted'
|
|
assert result['status'] == expected and result['run_id'] == run_id
|
|
assert journal.get(run_id) == before
|
|
assert result['result']['reconciled'] is True
|
|
assert reopened.events()[0]['event'] == 'result'
|
|
assert reopened.events()[0]['payload']['reconciled'] is True
|
|
assert not reopened.evaluate(service, now=NOW)[0]['dispatchable']
|
|
assert not list(data.rglob('command.tsv'))
|
|
|
|
|
|
def test_reconcile_refuses_active_owner_and_pending_resources(project_service, monkeypatch):
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids), hashes)
|
|
schedules.claim(row['schedule_id'], run_id='d' * 32)
|
|
with schedules.execution_lease(row['schedule_id']):
|
|
with pytest.raises(TaskCenterError, match='schedule_busy'):
|
|
ScheduleStore(data).reconcile_claim(service, row['schedule_id'])
|
|
monkeypatch.setattr(service, 'pending_batch_resources', lambda: [object()])
|
|
with pytest.raises(TaskCenterError, match='schedule_busy'):
|
|
schedules.reconcile_claim(service, row['schedule_id'])
|
|
assert schedules.get(row['schedule_id'])['status'] == 'claimed'
|
|
assert len(schedules.events()) == 1
|
|
|
|
|
|
def test_reconcile_audit_failure_leaves_claim_unchanged(project_service):
|
|
import sqlite3
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids), hashes)
|
|
schedules.claim(row['schedule_id'])
|
|
with sqlite3.connect(schedules.path) as db:
|
|
db.execute("CREATE TRIGGER reject_result BEFORE INSERT ON schedule_events BEGIN SELECT RAISE(ABORT, 'controlled fault'); END")
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
schedules.reconcile_claim(service, row['schedule_id'])
|
|
assert schedules.get(row['schedule_id'])['status'] == 'claimed'
|
|
|
|
|
|
def test_hard_process_exit_releases_schedule_lease_for_native_reconciliation(project_service):
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids, auto_start=True), hashes)
|
|
child = subprocess.run([sys.executable, '-c',
|
|
"import os,sys; sys.path.insert(0, 'src'); from dx_productivity.dsh.scheduling import ScheduleStore; "
|
|
"s=ScheduleStore(sys.argv[1]); lease=s.execution_lease(sys.argv[2]); lease.__enter__(); "
|
|
"s.claim(sys.argv[2],run_id='e'*32,automatic=True); os._exit(19)", str(data), row['schedule_id']],
|
|
cwd=Path(__file__).resolve().parents[2], capture_output=True, timeout=20)
|
|
assert child.returncode == 19, child.stderr.decode('utf8', 'replace')
|
|
assert ScheduleStore(data).get(row['schedule_id'])['status'] == 'claimed'
|
|
result = ScheduleStore(data).reconcile_claim(service, row['schedule_id'])
|
|
assert result['status'] == 'interrupted' and result['run_id'] == 'e' * 32
|
|
|
|
|
|
def test_active_claim_in_another_process_cannot_be_reconciled(project_service):
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
service, _, data, ids, hashes = project_service
|
|
schedules = ScheduleStore(data)
|
|
row = schedules.save(spec(ids, auto_start=True), hashes)
|
|
child = subprocess.Popen([sys.executable, '-c',
|
|
"import sys; sys.path.insert(0,'src'); from dx_productivity.dsh.scheduling import ScheduleStore; "
|
|
"s=ScheduleStore(sys.argv[1]); lease=s.execution_lease(sys.argv[2]); lease.__enter__(); "
|
|
"s.claim(sys.argv[2],run_id='f'*32,automatic=True); print('ready',flush=True); "
|
|
"sys.stdin.buffer.read(1); lease.__exit__(None,None,None)", str(data), row['schedule_id']],
|
|
cwd=Path(__file__).resolve().parents[2], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
try:
|
|
assert child.stdout.readline().strip() == b'ready'
|
|
with pytest.raises(TaskCenterError, match='schedule_busy'):
|
|
schedules.reconcile_claim(service, row['schedule_id'])
|
|
assert schedules.get(row['schedule_id'])['status'] == 'claimed'
|
|
finally:
|
|
child.communicate(input=b'q', timeout=10)
|
|
assert child.returncode == 0
|