Files
dxdesigner-part-replacement/tests/v1_1/test_batch_runtime.py
T

433 lines
19 KiB
Python

"""Batch resources are lazy, shared, and independently blocked after failed release."""
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from dx_productivity.trial_task_center.errors import TaskCenterError
class Desktop:
created = []
fail_release = False
def __init__(self, name, **_options):
self.name = name
self.processes = []
self.released = False
self.cleanup_record = None
def __enter__(self):
self.created.append(self)
return self
def start_process(self, command, **_options):
process = SimpleNamespace(pid=100 + len(self.processes), closed=False,
cleanup_record={}, returncode=0)
def close():
process.closed = True
process.cleanup_record = {'pid': process.pid, 'process_handle_closed': True,
'job_handle_closed': True, 'errors': []}
process.close = close
process.poll = lambda: process.returncode
self.processes.append(process)
return process
def close(self):
for process in self.processes:
process.close()
self.cleanup_record = {'desktop_handle_closed': True,
'desktop_confirmation': {'released': not self.fail_release}, 'errors': []}
if self.fail_release:
raise OSError('outside owner retains desktop')
self.released = True
@pytest.fixture(autouse=True)
def reset_desktop():
Desktop.created = []
Desktop.fail_release = False
def test_cache_only_batch_does_not_create_desktop_or_cad(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop):
pass
assert not Desktop.created
def test_native_operations_share_one_desktop_but_release_each_worker(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
names = []
for name in ('first', 'second'):
with runtime.native_scope(tmp_path / name) as scope:
names.append(scope.name)
process = scope.start_process(('inventory.exe',))
assert not process.closed
assert process.closed and not Desktop.created[0].released
assert names[0] == names[1]
assert len(Desktop.created) == 1 and Desktop.created[0].released
def test_failed_final_release_blocks_next_batch_and_resource_cleanup_never_replays(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import (
BatchCadRuntime, cleanup_pending_resources, guard_pending_resources, pending_resources)
Desktop.fail_release = True
with pytest.raises(TaskCenterError) as caught:
with BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) as runtime:
with runtime.native_scope(tmp_path / 'operation'):
pass
runtime.result = 'verified completed task'
assert caught.value.code == 'batch_resources_pending'
assert caught.value.completed_result == 'verified completed task'
assert pending_resources(tmp_path)
with pytest.raises(TaskCenterError):
guard_pending_resources(tmp_path)
assert not list(tmp_path.rglob('uncertain.flag'))
Desktop.fail_release = False
assert cleanup_pending_resources(tmp_path) == ()
guard_pending_resources(tmp_path)
assert len(Desktop.created) == 1
def test_cleanup_failure_takes_priority_over_cancel(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
Desktop.fail_release = True
with pytest.raises(TaskCenterError) as caught:
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
with runtime.native_scope(tmp_path / 'operation'):
pass
raise TaskCenterError('operation_cancelled', 'cancel requested')
assert caught.value.code == 'batch_resources_pending'
Desktop.fail_release = False
from dx_productivity.trial_task_center.batch_runtime import cleanup_pending_resources
cleanup_pending_resources(tmp_path)
class Cad:
created = []
def __init__(self, root, *, abort_event, desktop):
self.root, self.desktop, self.abort_event = root, desktop, abort_event
self.binding = SimpleNamespace(pid=500, guid='owned')
self.closed = False
def __enter__(self):
self.desktop.__enter__()
self.created.append(self)
return self
def checkpoint(self, evidence):
assert not self.closed
def start_worker(self, *args, **kwargs):
raise AssertionError('monitor is injected')
def __exit__(self, *_exc):
self.closed = True
self.desktop.close()
def command_for(tmp_path, name='one'):
root = tmp_path / 'data' / name
root.mkdir(parents=True)
project = tmp_path / ('project-' + name) / 'test.prj'
project.parent.mkdir()
project.write_text('project identity')
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\n'
f'task\tjob\t{project}\told\tnew\n')
return ('cscript', '//nologo', 'worker.vbs', str(root), 'python', 'job'), project
def worker_runtime(tmp_path, monkeypatch, monitor):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
from dx_productivity.trial_task_center import worker_monitor
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
Cad.created = []
return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad)
def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
one, _ = command_for(tmp_path)
two, _ = command_for(tmp_path, 'two')
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with runtime:
runtime.run_worker(one, 5)
assert project_checkpoint_released(Path(one[3]))
runtime.run_worker(two, 5)
assert project_checkpoint_released(Path(two[3]))
assert len(Cad.created) == len(Desktop.created) == 1
assert Cad.created[0].closed and Desktop.created[0].released
@pytest.mark.parametrize('second_fails', [False, True])
def test_prewrite_failure_releases_old_desktop_and_retries_only_once(tmp_path, monkeypatch, second_fails):
import subprocess
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(command, *args, **kwargs):
calls.append(len(calls))
if len(calls) == 2:
assert Desktop.created[0].released
return subprocess.CompletedProcess(command, 2 if len(calls) == 1 or second_fails else 0, '', 'startup')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime:
if second_fails:
with pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
else:
runtime.run_worker(command, 5)
assert len(calls) == len(Desktop.created) == len(Cad.created) == 2
assert runtime.generation == 2
def test_possible_mutation_prevents_automatic_replay(tmp_path, monkeypatch):
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(*args, **kwargs):
calls.append(1)
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
record = json.loads(path.read_text())
record['state'] = 'mutation_possible'
path.write_text(json.dumps(record))
raise WorkerMonitorError('lost connection after possible write')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime, pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
assert len(calls) == len(Desktop.created) == 1
def test_previous_checkpoint_invalidated_before_new_worker(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(*args, **kwargs):
assert not project_checkpoint_released(Path(command[3]))
calls.append(1)
return subprocess.CompletedProcess(command, 0, '', '')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime:
runtime.run_worker(command, 5)
assert project_checkpoint_released(Path(command[3]))
def fail_checkpoint(*args):
raise WorkerMonitorError('cannot certify empty project')
runtime.cad.checkpoint = fail_checkpoint
# No retry after evidence has been marked as possibly modified.
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
record = json.loads(path.read_text())
record['state'] = 'mutation_possible'
path.write_text(json.dumps(record))
with pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
assert not project_checkpoint_released(Path(command[3]))
def test_attempt_publication_failure_never_launches_worker(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_attempt
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: pytest.fail('must not launch'))
def fail(*args, **kwargs):
raise OSError('disk unavailable')
monkeypatch.setattr(batch_attempt, 'prepare_attempt', fail)
with runtime, pytest.raises(OSError):
runtime.run_worker(command, 5)
assert not Cad.created
def test_boundary_pause_does_not_abort_current_cad_startup(tmp_path, monkeypatch):
import subprocess
from threading import Event
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
pause, worker_abort = Event(), Event()
runtime.stop_event = pause
original_enter = Cad.__enter__
def enter(cad):
pause.set() # User requests pause after the current task has begun.
assert cad.abort_event is worker_abort
assert not cad.abort_event.is_set()
return original_enter(cad)
monkeypatch.setattr(Cad, '__enter__', enter)
with runtime:
runtime.run_worker(command, 5, abort_event=worker_abort)
assert not list(tmp_path.rglob('uncertain.flag'))
def test_cleanup_still_runs_when_resource_journal_write_fails(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_runtime
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
with pytest.raises(TaskCenterError) as caught:
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
real_write = batch_runtime.write_record
def failing_write(path, record):
if record['state'] == 'closing':
raise OSError('journal unavailable')
return real_write(path, record)
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
assert Desktop.created[0].released
assert caught.value.code == 'batch_resources_pending'
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
runtime.close()
def test_real_startup_failure_with_released_lease_can_retry(tmp_path, monkeypatch):
import io
import subprocess
from dx_productivity.trial_task_center import isolated_cad, worker_monitor
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
command, _ = command_for(tmp_path)
starts = []
def executable():
starts.append(1)
if len(starts) == 1:
raise worker_monitor.WorkerMonitorError('registry temporarily unavailable')
return 'cad.exe'
class BootstrapDesktop(Desktop):
def start_process(self, command, **options):
process = super().start_process(command, **options)
process.stderr = io.BytesIO()
process.wait = lambda timeout: process.returncode
if command[0] == 'cad.exe':
self.cad = process
process.returncode = None
else:
action, evidence, pid = command
environment = options['env']
binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned',
'token': environment['DXPART_SESSION_TOKEN']}
if action == 'probe':
Path(evidence).write_text(json.dumps(binding))
elif action == 'checkpoint':
Path(evidence).write_text(json.dumps({'binding': binding, 'project_path': '', 'project_closed': True}))
elif action == 'quit':
self.cad.returncode = 0
return process
monkeypatch.setattr(isolated_cad, '_designer_executable', executable)
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args)
monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=BootstrapDesktop) as runtime:
runtime.run_worker(command, 5)
assert len(starts) == len(Desktop.created) == 2
assert all(desktop.released for desktop in Desktop.created)
def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path):
import hashlib
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
command, project = command_for(tmp_path)
database = project.parent / 'database.dat'
database.write_bytes(b'database')
evidence = Path(command[3]) / 'native'
evidence.mkdir()
proof = {'version': 1, 'native_read_only': True, 'workers_closed': True,
'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()),
'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest()
for path in (project, database)}}
path = evidence / 'native-retry-proof.json'
path.write_text(json.dumps(proof))
with BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) as runtime:
with runtime.native_scope(evidence) as scope:
pass
proof['binding'] = scope.identity
path.write_text(json.dumps(proof))
database.write_bytes(b'changed externally')
assert not runtime.recover_native_failure(project, evidence, 'timeout')
database.write_bytes(b'database')
assert runtime.recover_native_failure(project, evidence, 'timeout')
assert Desktop.created[0].released
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
with pytest.raises(WorkerMonitorError, match='stale'):
scope.start_process(('late-old-worker.exe',))
assert scope.identity['generation'] == 1 and runtime.generation == 2
scope.close() # A late close cannot grant the old operation a new identity.
assert str(evidence.resolve()) not in runtime.record['native_operations']
assert not runtime.recover_native_failure(project, evidence, 'another timeout')
assert len(Desktop.created) == 2
def test_release_publication_failure_can_be_reconciled_without_replaying(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_runtime
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
real_write = batch_runtime.write_record
def failing_write(path, record):
if record['state'] == 'released':
raise OSError('cannot publish release')
return real_write(path, record)
with pytest.raises(TaskCenterError):
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
assert Desktop.created[0].released and batch_runtime.pending_resources(tmp_path)
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
assert batch_runtime.cleanup_pending_resources(tmp_path) == ()
assert len(Desktop.created) == 1
@pytest.mark.parametrize('record', [
{'version': 1, 'session_id': 'old', 'state': 'blocked'},
{'version': 1, 'session_id': 'old', 'state': 'released'},
[],
])
def test_unknown_resource_evidence_never_authorizes_cleanup(tmp_path, record):
from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources, pending_resources
path = tmp_path / 'batch-resources' / 'old' / 'session.json'
path.parent.mkdir(parents=True)
path.write_text(json.dumps(record))
assert pending_resources(tmp_path)
assert cleanup_pending_resources(tmp_path)
assert json.loads(path.read_text()) == record
def test_stale_generation_and_unknown_checkpoint_schema_are_rejected(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with runtime:
runtime.run_worker(command, 5)
path = Path(command[3]) / 'project-checkpoint.json'
checkpoint = json.loads(path.read_text())
assert project_checkpoint_released(Path(command[3]))
for changes in ({'version': 999}, {'generation': 999}, {'attempt_id': 'old'}, {'cad_guid': 'foreign'}):
path.write_text(json.dumps(dict(checkpoint, **changes)))
assert not project_checkpoint_released(Path(command[3]))
def test_restart_reconciles_only_confirmed_disappearance_without_replacement(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_resources, isolated_desktop
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
Desktop.fail_release = True
runtime = BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
with pytest.raises(TaskCenterError):
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
# Simulate restarting the UI: there are no retained owned handles to act on.
batch_resources.LIVE_RESOURCES.pop(str(runtime.path))
present = [True]
class Api:
def confirm_desktop_released(self, name):
assert name == runtime.record['desktop']
if present[0]:
raise OSError('desktop still held by external thread')
monkeypatch.setattr(isolated_desktop, '_Win32Api', Api)
assert batch_resources.cleanup_pending_resources(tmp_path)
present[0] = False
assert batch_resources.cleanup_pending_resources(tmp_path) == ()
assert len(Desktop.created) == 1