677 lines
31 KiB
Python
677 lines
31 KiB
Python
"""Batch resources are lazy, shared, and independently blocked after failed release."""
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
|
|
|
|
class Desktop:
|
|
created = []
|
|
fail_release = False
|
|
|
|
def __init__(self, name, **_options):
|
|
self.name = name
|
|
self.processes = []
|
|
self.released = False
|
|
self.cleanup_record = None
|
|
|
|
def __enter__(self):
|
|
self.created.append(self)
|
|
return self
|
|
|
|
def start_process(self, command, **_options):
|
|
process = SimpleNamespace(pid=100 + len(self.processes), closed=False,
|
|
cleanup_record={}, returncode=0)
|
|
def close():
|
|
process.closed = True
|
|
process.cleanup_record = {'pid': process.pid, 'process_handle_closed': True,
|
|
'job_handle_closed': True, 'errors': []}
|
|
process.close = close
|
|
process.poll = lambda: process.returncode
|
|
self.processes.append(process)
|
|
return process
|
|
|
|
def close(self):
|
|
for process in self.processes:
|
|
process.close()
|
|
self.cleanup_record = {'desktop_handle_closed': True,
|
|
'desktop_confirmation': {'released': not self.fail_release}, 'errors': []}
|
|
if self.fail_release:
|
|
raise OSError('outside owner retains desktop')
|
|
self.released = True
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_desktop():
|
|
Desktop.created = []
|
|
Desktop.fail_release = False
|
|
|
|
|
|
def test_cache_only_batch_does_not_create_desktop_or_cad(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop):
|
|
pass
|
|
assert not Desktop.created
|
|
|
|
|
|
def test_native_operations_share_one_desktop_but_release_each_worker(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
|
|
names = []
|
|
for name in ('first', 'second'):
|
|
with runtime.native_scope(tmp_path / name) as scope:
|
|
names.append(scope.name)
|
|
process = scope.start_process(('inventory.exe',))
|
|
assert not process.closed
|
|
assert process.closed and not Desktop.created[0].released
|
|
assert names[0] == names[1]
|
|
assert len(Desktop.created) == 1 and Desktop.created[0].released
|
|
|
|
|
|
def test_failed_final_release_blocks_next_batch_and_resource_cleanup_never_replays(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import (
|
|
BatchCadRuntime, cleanup_pending_resources, guard_pending_resources, pending_resources)
|
|
Desktop.fail_release = True
|
|
with pytest.raises(TaskCenterError) as caught:
|
|
with BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) as runtime:
|
|
with runtime.native_scope(tmp_path / 'operation'):
|
|
pass
|
|
runtime.result = 'verified completed task'
|
|
assert caught.value.code == 'batch_resources_pending'
|
|
assert caught.value.completed_result == 'verified completed task'
|
|
assert pending_resources(tmp_path)
|
|
with pytest.raises(TaskCenterError):
|
|
guard_pending_resources(tmp_path)
|
|
assert not list(tmp_path.rglob('uncertain.flag'))
|
|
Desktop.fail_release = False
|
|
assert cleanup_pending_resources(tmp_path) == ()
|
|
guard_pending_resources(tmp_path)
|
|
assert len(Desktop.created) == 1
|
|
|
|
|
|
def test_cleanup_failure_takes_priority_over_cancel(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
Desktop.fail_release = True
|
|
with pytest.raises(TaskCenterError) as caught:
|
|
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
|
|
with runtime.native_scope(tmp_path / 'operation'):
|
|
pass
|
|
raise TaskCenterError('operation_cancelled', 'cancel requested')
|
|
assert caught.value.code == 'batch_resources_pending'
|
|
Desktop.fail_release = False
|
|
from dx_productivity.trial_task_center.batch_runtime import cleanup_pending_resources
|
|
cleanup_pending_resources(tmp_path)
|
|
|
|
|
|
class Cad:
|
|
created = []
|
|
|
|
def __init__(self, root, *, abort_event, desktop):
|
|
self.root, self.desktop, self.abort_event = root, desktop, abort_event
|
|
self.binding = SimpleNamespace(pid=500, guid='owned')
|
|
self.closed = False
|
|
|
|
def __enter__(self):
|
|
self.desktop.__enter__()
|
|
self.created.append(self)
|
|
return self
|
|
|
|
def checkpoint(self, evidence):
|
|
assert not self.closed
|
|
|
|
def start_worker(self, *args, **kwargs):
|
|
raise AssertionError('monitor is injected')
|
|
|
|
def __exit__(self, *_exc):
|
|
self.closed = True
|
|
self.desktop.close()
|
|
|
|
|
|
def command_for(tmp_path, name='one'):
|
|
root = tmp_path / 'data' / name
|
|
root.mkdir(parents=True)
|
|
project = tmp_path / ('project-' + name) / 'test.prj'
|
|
project.parent.mkdir()
|
|
project.write_text('project identity')
|
|
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\n'
|
|
f'task\tjob\t{project}\told\tnew\n')
|
|
return ('cscript', '//nologo', 'worker.vbs', str(root), 'python', 'job'), project
|
|
|
|
|
|
def worker_runtime(tmp_path, monkeypatch, monitor):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
from dx_productivity.trial_task_center import worker_monitor
|
|
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
|
|
Cad.created = []
|
|
return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad)
|
|
|
|
|
|
def test_same_project_reuses_cad_then_next_project_starts_a_fresh_session(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
|
one, _ = command_for(tmp_path)
|
|
two, _ = command_for(tmp_path, 'two')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(one, 5)
|
|
assert project_checkpoint_released(Path(one[3]))
|
|
first = runtime.cad
|
|
runtime.run_worker(one, 5)
|
|
assert runtime.cad is first
|
|
runtime.run_worker(two, 5)
|
|
assert project_checkpoint_released(Path(two[3]))
|
|
assert first.closed and Desktop.created[0].released
|
|
assert len(Cad.created) == len(Desktop.created) == 2
|
|
assert all(cad.closed for cad in Cad.created)
|
|
assert all(desktop.released for desktop in Desktop.created)
|
|
|
|
|
|
def test_completed_project_does_not_hold_resolution_wait_for_next_project_failure(tmp_path, monkeypatch):
|
|
import subprocess
|
|
command, _ = command_for(tmp_path)
|
|
next_command, _ = command_for(tmp_path, 'next')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(command, 5)
|
|
assert runtime._active_attempts == []
|
|
def fail_prepare(*args, **kwargs):
|
|
raise OSError('next project backup failed')
|
|
monkeypatch.setattr(runtime, '_prepare_attempts', fail_prepare)
|
|
with pytest.raises(OSError, match='next project backup failed'):
|
|
runtime.run_worker(next_command, 5)
|
|
assert not getattr(runtime.stop_event, 'waiting_for_resolution', False)
|
|
assert not (Path(command[3]) / 'quick-run' / 'uncertain.flag').exists()
|
|
|
|
|
|
def test_current_task_and_attempt_are_durable_before_worker_launch(tmp_path, monkeypatch):
|
|
import subprocess
|
|
one, _ = command_for(tmp_path)
|
|
two, _ = command_for(tmp_path, 'next')
|
|
next_root = Path(two[3])
|
|
command_path = next_root / 'command.tsv'
|
|
command_path.write_text(command_path.read_text().replace('task\tjob\t', 'task2\tjob2\t'), encoding='utf-8')
|
|
calls = []
|
|
def monitor(command, *args, **kwargs):
|
|
calls.append(command)
|
|
record = json.loads(runtime.path.read_text(encoding='utf-8'))
|
|
expected_task = 'task' if len(calls) == 1 else 'task2'
|
|
assert record.get('active_task') == expected_task
|
|
assert record.get('active_attempt')
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
two = (*two[:-1], 'job2')
|
|
with runtime:
|
|
runtime.run_worker(one, 5)
|
|
runtime.run_worker(two, 5)
|
|
|
|
|
|
@pytest.mark.parametrize('second_fails', [False, True])
|
|
def test_prewrite_failure_releases_old_desktop_and_retries_only_once(tmp_path, monkeypatch, second_fails):
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
command, _ = command_for(tmp_path)
|
|
calls = []
|
|
def monitor(command, *args, **kwargs):
|
|
calls.append(len(calls))
|
|
if len(calls) == 2:
|
|
assert Desktop.created[0].released
|
|
return subprocess.CompletedProcess(command, 2 if len(calls) == 1 or second_fails else 0, '', 'startup')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
with runtime:
|
|
if second_fails:
|
|
with pytest.raises(WorkerMonitorError):
|
|
runtime.run_worker(command, 5)
|
|
else:
|
|
runtime.run_worker(command, 5)
|
|
assert len(calls) == len(Desktop.created) == len(Cad.created) == 2
|
|
assert runtime.generation == 2
|
|
releases = json.loads(runtime.path.read_text())['generation_releases']
|
|
assert [release['generation'] for release in releases] == [1, 2]
|
|
assert [release['desktop'] for release in releases] == [desktop.name for desktop in Desktop.created]
|
|
assert all(release['cad_pid'] == 500 and release['cad_guid'] == 'owned'
|
|
and release['cleanup']['desktop_confirmation']['released'] is True
|
|
and release['released_at'] > 0 for release in releases)
|
|
|
|
|
|
def test_rebuild_does_not_hide_cad_shutdown_failure(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
|
|
class FailingCad(Cad):
|
|
def __exit__(self, *_exc):
|
|
self.closed = True
|
|
self.desktop.close()
|
|
raise OSError('CAD quit confirmation failed')
|
|
|
|
runtime = BatchCadRuntime(tmp_path / 'data', 'execution',
|
|
desktop_factory=Desktop, cad_factory=FailingCad)
|
|
runtime.__enter__()
|
|
runtime._ensure_desktop()
|
|
runtime._ensure_cad()
|
|
with pytest.raises(TaskCenterError, match='CAD quit confirmation failed'):
|
|
runtime.close(rebuilding=True)
|
|
with pytest.raises(TaskCenterError, match='CAD quit confirmation failed'):
|
|
runtime.__exit__(None, None, None)
|
|
record = json.loads(runtime.path.read_text())
|
|
assert record['state'] == 'blocked'
|
|
|
|
|
|
def test_possible_mutation_prevents_automatic_replay(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
command, _ = command_for(tmp_path)
|
|
calls = []
|
|
def monitor(*args, **kwargs):
|
|
calls.append(1)
|
|
path = next((runtime.path.parent / 'attempts').rglob('job.json'))
|
|
record = json.loads(path.read_text())
|
|
record['state'] = 'mutation_possible'
|
|
path.write_text(json.dumps(record))
|
|
raise WorkerMonitorError('lost connection after possible write')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
with runtime, pytest.raises(WorkerMonitorError):
|
|
runtime.run_worker(command, 5)
|
|
assert len(calls) == len(Desktop.created) == 1
|
|
|
|
|
|
def test_previous_checkpoint_invalidated_before_new_worker(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
command, _ = command_for(tmp_path)
|
|
calls = []
|
|
def monitor(*args, **kwargs):
|
|
assert not project_checkpoint_released(Path(command[3]))
|
|
calls.append(1)
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
with runtime:
|
|
runtime.run_worker(command, 5)
|
|
assert project_checkpoint_released(Path(command[3]))
|
|
def fail_checkpoint(*args):
|
|
raise WorkerMonitorError('cannot certify empty project')
|
|
runtime.cad.checkpoint = fail_checkpoint
|
|
# No retry after evidence has been marked as possibly modified.
|
|
path = next((runtime.path.parent / 'attempts').rglob('job.json'))
|
|
record = json.loads(path.read_text())
|
|
record['state'] = 'mutation_possible'
|
|
path.write_text(json.dumps(record))
|
|
with pytest.raises(WorkerMonitorError):
|
|
runtime.run_worker(command, 5)
|
|
assert not project_checkpoint_released(Path(command[3]))
|
|
|
|
|
|
def test_attempt_publication_failure_never_launches_worker(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import batch_attempt
|
|
command, _ = command_for(tmp_path)
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: pytest.fail('must not launch'))
|
|
def fail(*args, **kwargs):
|
|
raise OSError('disk unavailable')
|
|
monkeypatch.setattr(batch_attempt, 'prepare_attempt', fail)
|
|
with runtime, pytest.raises(OSError):
|
|
runtime.run_worker(command, 5)
|
|
assert not Cad.created
|
|
assert not Desktop.created
|
|
from dx_productivity.trial_task_center.batch_resources import pending_resources
|
|
assert pending_resources(runtime.root) == ()
|
|
|
|
|
|
def test_boundary_pause_does_not_abort_current_cad_startup(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from threading import Event
|
|
command, _ = command_for(tmp_path)
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
pause, worker_abort = Event(), Event()
|
|
runtime.stop_event = pause
|
|
original_enter = Cad.__enter__
|
|
def enter(cad):
|
|
pause.set() # User requests pause after the current task has begun.
|
|
assert cad.abort_event is worker_abort
|
|
assert not cad.abort_event.is_set()
|
|
return original_enter(cad)
|
|
monkeypatch.setattr(Cad, '__enter__', enter)
|
|
with runtime:
|
|
runtime.run_worker(command, 5, abort_event=worker_abort)
|
|
assert not list(tmp_path.rglob('uncertain.flag'))
|
|
|
|
|
|
def test_cleanup_still_runs_when_resource_journal_write_fails(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import batch_runtime
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
|
with pytest.raises(TaskCenterError) as caught:
|
|
with runtime:
|
|
with runtime.native_scope(tmp_path / 'read'):
|
|
pass
|
|
real_write = batch_runtime.write_record
|
|
def failing_write(path, record):
|
|
if record['state'] == 'closing':
|
|
raise OSError('journal unavailable')
|
|
return real_write(path, record)
|
|
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
|
|
assert Desktop.created[0].released
|
|
assert caught.value.code == 'batch_resources_pending'
|
|
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
|
|
runtime.close()
|
|
|
|
|
|
def test_real_startup_failure_with_released_lease_can_retry(tmp_path, monkeypatch):
|
|
import io
|
|
import subprocess
|
|
from dx_productivity.trial_task_center import isolated_cad, worker_monitor
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
command, _ = command_for(tmp_path)
|
|
starts = []
|
|
def executable():
|
|
starts.append(1)
|
|
if len(starts) == 1:
|
|
raise worker_monitor.WorkerMonitorError('registry temporarily unavailable')
|
|
return 'cad.exe'
|
|
class BootstrapDesktop(Desktop):
|
|
def start_process(self, command, **options):
|
|
process = super().start_process(command, **options)
|
|
process.stderr = io.BytesIO()
|
|
process.wait = lambda timeout: process.returncode
|
|
if command[0] == 'cad.exe':
|
|
self.cad = process
|
|
process.returncode = None
|
|
else:
|
|
action, evidence, pid = command
|
|
environment = options['env']
|
|
binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned',
|
|
'token': environment['DXPART_SESSION_TOKEN']}
|
|
if action == 'probe':
|
|
Path(evidence).write_text(json.dumps(binding))
|
|
elif action == 'checkpoint':
|
|
Path(evidence).write_text(json.dumps({'binding': binding, 'project_path': '', 'project_closed': True}))
|
|
elif action == 'quit':
|
|
self.cad.returncode = 0
|
|
return process
|
|
monkeypatch.setattr(isolated_cad, '_designer_executable', executable)
|
|
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args)
|
|
monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=BootstrapDesktop) as runtime:
|
|
runtime.run_worker(command, 5)
|
|
assert len(starts) == len(Desktop.created) == 2
|
|
assert all(desktop.released for desktop in Desktop.created)
|
|
|
|
|
|
def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path):
|
|
import hashlib
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
command, project = command_for(tmp_path)
|
|
database = project.parent / 'database.dat'
|
|
database.write_bytes(b'database')
|
|
evidence = Path(command[3]) / 'native'
|
|
evidence.mkdir()
|
|
proof = {'version': 1, 'native_read_only': True, 'workers_closed': True,
|
|
'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()),
|
|
'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in (project, database)}}
|
|
path = evidence / 'native-retry-proof.json'
|
|
path.write_text(json.dumps(proof))
|
|
with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop) as runtime:
|
|
with runtime.native_scope(evidence) as scope:
|
|
pass
|
|
proof['binding'] = scope.identity
|
|
path.write_text(json.dumps(proof))
|
|
database.write_bytes(b'changed externally')
|
|
assert not runtime.recover_native_failure(project, evidence, 'timeout')
|
|
database.write_bytes(b'database')
|
|
assert runtime.recover_native_failure(project, evidence, 'timeout')
|
|
assert Desktop.created[0].released
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
with pytest.raises(WorkerMonitorError, match='stale'):
|
|
scope.start_process(('late-old-worker.exe',))
|
|
assert scope.identity['generation'] == 1 and runtime.generation == 2
|
|
scope.close() # A late close cannot grant the old operation a new identity.
|
|
assert str(evidence.resolve()) not in runtime.record['native_operations']
|
|
assert not runtime.recover_native_failure(project, evidence, 'another timeout')
|
|
assert len(Desktop.created) == 2
|
|
|
|
|
|
def test_mock_native_read_recovery_spends_same_project_worker_retry_budget(tmp_path, monkeypatch):
|
|
"""Real durable proof/runtime logic; desktop, CAD and worker failure are fakes."""
|
|
import hashlib
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
command, project = command_for(tmp_path)
|
|
database = project.parent / 'database.dat'
|
|
database.write_bytes(b'unchanged database')
|
|
evidence = Path(command[3]) / 'native'
|
|
evidence.mkdir()
|
|
calls = []
|
|
def fail_worker(*args, **kwargs):
|
|
calls.append(1)
|
|
raise WorkerMonitorError('prewrite failure after native recovery')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, fail_worker)
|
|
with runtime:
|
|
with runtime.native_scope(evidence) as scope:
|
|
pass
|
|
proof = {'version': 1, 'native_read_only': True, 'workers_closed': True,
|
|
'binding': scope.identity, 'project_path': str(project.resolve()),
|
|
'evidence_root': str(evidence.resolve()),
|
|
'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in (project, database)}}
|
|
(evidence / 'native-retry-proof.json').write_text(json.dumps(proof))
|
|
assert runtime.recover_native_failure(project, evidence, 'read timeout')
|
|
assert len(Desktop.created) == 2 and Desktop.created[0].released
|
|
with pytest.raises(WorkerMonitorError, match='after native recovery'):
|
|
runtime.run_worker(command, 5)
|
|
assert calls == [1]
|
|
assert len(Desktop.created) == 2
|
|
assert not runtime.recover_native_failure(project, evidence, 'another read timeout')
|
|
assert all(desktop.released for desktop in Desktop.created)
|
|
assert runtime.record['retries'][str(project.resolve()).casefold()]['count'] == 1
|
|
|
|
|
|
def test_mock_different_projects_each_get_one_retry_without_accumulating_budget(tmp_path, monkeypatch):
|
|
"""Real per-project accounting; fake CAD/desktop and unchanged prewrite failures."""
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
one, project_one = command_for(tmp_path)
|
|
two, project_two = command_for(tmp_path, 'two')
|
|
calls = {one[3]: 0, two[3]: 0}
|
|
def monitor(command, *args, **kwargs):
|
|
calls[command[3]] += 1
|
|
if calls[command[3]] == 2:
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
return subprocess.CompletedProcess(command, 2, '', 'unchanged prewrite failure')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
with runtime:
|
|
runtime.run_worker(one, 5)
|
|
assert len(Desktop.created) == 2
|
|
runtime.run_worker(two, 5)
|
|
assert len(Desktop.created) == 4 # Separate project startup plus its single rebuild.
|
|
for expected_sessions, command in enumerate((one, two), 5):
|
|
with pytest.raises(WorkerMonitorError, match='prewrite failure'):
|
|
runtime.run_worker(command, 5)
|
|
assert len(Desktop.created) == expected_sessions # Switching projects is not another retry.
|
|
assert calls == {one[3]: 3, two[3]: 3}
|
|
assert set(runtime.record['retries']) == {
|
|
str(project_one.resolve()).casefold(), str(project_two.resolve()).casefold()}
|
|
assert all(value['count'] == 1 for value in runtime.record['retries'].values())
|
|
assert all(desktop.released for desktop in Desktop.created)
|
|
|
|
|
|
def test_release_publication_failure_can_be_reconciled_without_replaying(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import batch_runtime
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
|
real_write = batch_runtime.write_record
|
|
def failing_write(path, record):
|
|
if record['state'] == 'released':
|
|
raise OSError('cannot publish release')
|
|
return real_write(path, record)
|
|
with pytest.raises(TaskCenterError):
|
|
with runtime:
|
|
with runtime.native_scope(tmp_path / 'read'):
|
|
pass
|
|
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
|
|
assert Desktop.created[0].released and batch_runtime.pending_resources(tmp_path)
|
|
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
|
|
assert batch_runtime.cleanup_pending_resources(tmp_path) == ()
|
|
assert len(Desktop.created) == 1
|
|
releases = json.loads(runtime.path.read_text())['generation_releases']
|
|
assert len(releases) == 1 and releases[0]['generation'] == 1
|
|
|
|
|
|
@pytest.mark.parametrize('record', [
|
|
{'version': 1, 'session_id': 'old', 'state': 'blocked'},
|
|
{'version': 1, 'session_id': 'old', 'state': 'released'},
|
|
[],
|
|
])
|
|
def test_unknown_resource_evidence_never_authorizes_cleanup(tmp_path, record):
|
|
from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources, pending_resources
|
|
path = tmp_path / 'batch-resources' / 'old' / 'session.json'
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(json.dumps(record))
|
|
assert pending_resources(tmp_path)
|
|
assert cleanup_pending_resources(tmp_path)
|
|
assert json.loads(path.read_text()) == record
|
|
|
|
|
|
def test_stale_generation_and_unknown_checkpoint_schema_are_rejected(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
|
command, _ = command_for(tmp_path)
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(command, 5)
|
|
path = Path(command[3]) / 'project-checkpoint.json'
|
|
checkpoint = json.loads(path.read_text())
|
|
assert project_checkpoint_released(Path(command[3]))
|
|
for changes in ({'version': 999}, {'generation': 999}, {'attempt_id': 'old'}, {'cad_guid': 'foreign'}):
|
|
path.write_text(json.dumps(dict(checkpoint, **changes)))
|
|
assert not project_checkpoint_released(Path(command[3]))
|
|
|
|
|
|
def test_restart_reconciles_only_confirmed_disappearance_without_replacement(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import batch_resources, isolated_desktop
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
Desktop.fail_release = True
|
|
runtime = BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
|
with pytest.raises(TaskCenterError):
|
|
with runtime:
|
|
with runtime.native_scope(tmp_path / 'read'):
|
|
pass
|
|
# Simulate restarting the UI: there are no retained owned handles to act on.
|
|
batch_resources.LIVE_RESOURCES.pop(str(runtime.path))
|
|
present = [True]
|
|
class Api:
|
|
def confirm_desktop_released(self, name):
|
|
assert name == runtime.record['desktop']
|
|
if present[0]:
|
|
raise OSError('desktop still held by external thread')
|
|
monkeypatch.setattr(isolated_desktop, '_Win32Api', Api)
|
|
assert batch_resources.cleanup_pending_resources(tmp_path)
|
|
present[0] = False
|
|
assert batch_resources.cleanup_pending_resources(tmp_path) == ()
|
|
assert len(Desktop.created) == 1
|
|
|
|
|
|
@pytest.mark.parametrize('conflict', [
|
|
'bootstrap_binding', 'bootstrap_json', 'bootstrap_schema', 'helper_identity',
|
|
'checkpoint_binding', 'checkpoint_json', 'worker_identity',
|
|
])
|
|
def test_identity_conflict_never_rebuilds_or_launches_another_worker(tmp_path, monkeypatch, conflict):
|
|
import io
|
|
import subprocess
|
|
from dx_productivity.trial_task_center import isolated_cad, worker_monitor
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
command, _ = command_for(tmp_path)
|
|
workers = []
|
|
|
|
class IdentityDesktop(Desktop):
|
|
def start_process(self, launch, **options):
|
|
process = super().start_process(launch, **options)
|
|
process.stderr = io.BytesIO(b'CAD COM GUID does not match its owned session')
|
|
process.wait = lambda timeout: process.returncode
|
|
if launch[0] == 'cad.exe':
|
|
self.cad = process
|
|
process.returncode = None
|
|
return process
|
|
action, evidence, pid = launch
|
|
binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned',
|
|
'token': options['env']['DXPART_SESSION_TOKEN']}
|
|
if action == 'probe':
|
|
if conflict == 'helper_identity':
|
|
process.returncode = 4 # Private helper identity-conflict exit.
|
|
return process
|
|
if conflict == 'bootstrap_binding':
|
|
binding['pid'] = pid + 1
|
|
if conflict == 'bootstrap_schema':
|
|
binding.pop('token')
|
|
Path(evidence).write_text('{' if conflict == 'bootstrap_json' else json.dumps(binding))
|
|
elif action == 'checkpoint':
|
|
closing = Path(evidence).name.startswith('cad-closed-')
|
|
if closing and conflict == 'checkpoint_binding':
|
|
binding['guid'] = 'foreign'
|
|
record = {'binding': binding, 'project_path': '', 'project_closed': True}
|
|
Path(evidence).write_text('{' if closing and conflict == 'checkpoint_json' else json.dumps(record))
|
|
elif action == 'quit':
|
|
self.cad.returncode = 0
|
|
return process
|
|
|
|
monkeypatch.setattr(isolated_cad, '_designer_executable', lambda: 'cad.exe')
|
|
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args)
|
|
def monitor(*args, **kwargs):
|
|
workers.append(1)
|
|
if conflict == 'worker_identity':
|
|
return subprocess.CompletedProcess(command, 2, '', 'private CAD GUID mismatch; no project was changed')
|
|
return subprocess.CompletedProcess(command, 0, '', '')
|
|
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
|
|
runtime = BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=IdentityDesktop)
|
|
with runtime, pytest.raises(worker_monitor.WorkerMonitorError):
|
|
runtime.run_worker(command, 5)
|
|
assert len(Desktop.created) == 1
|
|
assert len(workers) == (1 if conflict.startswith('checkpoint') or conflict == 'worker_identity' else 0)
|
|
assert runtime.generation == 1 and not runtime.record['retries']
|
|
assert Desktop.created[0].released
|
|
|
|
|
|
def test_request_session_identity_is_uuid_bound(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
session = '12345678-1234-5678-1234-567812345678'
|
|
runtime = BatchCadRuntime(tmp_path, 'bom_import', session_id=session)
|
|
assert runtime.session_id == session.replace('-', '')
|
|
with pytest.raises(ValueError):
|
|
BatchCadRuntime(tmp_path, 'bom_import', session_id='../escape')
|
|
|
|
|
|
def test_cleanup_preserves_primary_bom_error_and_report(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
|
primary = TaskCenterError('bom_inspection', 'read failed')
|
|
primary.bom_report = object()
|
|
Desktop.fail_release = True
|
|
with pytest.raises(TaskCenterError) as caught:
|
|
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
|
|
with runtime.native_scope(tmp_path / 'operation'):
|
|
pass
|
|
raise primary
|
|
error = caught.value
|
|
assert error.primary_error is primary
|
|
assert error.cleanup_error is not None
|
|
assert error.bom_report is primary.bom_report
|
|
assert 'BOM' in error.message and '已核验任务结果保留' not in error.message
|
|
assert runtime.record['primary_error']['code'] == primary.code
|
|
assert runtime.record['cleanup_error']
|
|
Desktop.fail_release = False
|
|
from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources
|
|
cleanup_pending_resources(tmp_path)
|
|
|
|
|
|
def test_attempts_live_under_short_session_root(tmp_path, monkeypatch):
|
|
import subprocess
|
|
command, _ = command_for(tmp_path)
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
|
subprocess.CompletedProcess(command, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(command, 5)
|
|
assert list((runtime.path.parent / 'attempts').rglob('job.json'))
|
|
assert not (Path(command[3]) / 'batch-attempts').exists()
|