"""Batch resources are lazy, shared, and independently blocked after failed release.""" import json from pathlib import Path from types import SimpleNamespace import pytest from dx_productivity.trial_task_center.errors import TaskCenterError class Desktop: created = [] fail_release = False def __init__(self, name, **_options): self.name = name self.processes = [] self.released = False self.cleanup_record = None def __enter__(self): self.created.append(self) return self def start_process(self, command, **_options): process = SimpleNamespace(pid=100 + len(self.processes), closed=False, cleanup_record={}, returncode=0) def close(): process.closed = True process.cleanup_record = {'pid': process.pid, 'process_handle_closed': True, 'job_handle_closed': True, 'errors': []} process.close = close process.poll = lambda: process.returncode self.processes.append(process) return process def close(self): for process in self.processes: process.close() self.cleanup_record = {'desktop_handle_closed': True, 'desktop_confirmation': {'released': not self.fail_release}, 'errors': []} if self.fail_release: raise OSError('outside owner retains desktop') self.released = True @pytest.fixture(autouse=True) def reset_desktop(): Desktop.created = [] Desktop.fail_release = False def test_cache_only_batch_does_not_create_desktop_or_cad(tmp_path): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop): pass assert not Desktop.created def test_native_operations_share_one_desktop_but_release_each_worker(tmp_path): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime: names = [] for name in ('first', 'second'): with runtime.native_scope(tmp_path / name) as scope: names.append(scope.name) process = scope.start_process(('inventory.exe',)) assert not process.closed assert process.closed and not Desktop.created[0].released assert names[0] == names[1] assert len(Desktop.created) == 1 and Desktop.created[0].released def test_failed_final_release_blocks_next_batch_and_resource_cleanup_never_replays(tmp_path): from dx_productivity.trial_task_center.batch_runtime import ( BatchCadRuntime, cleanup_pending_resources, guard_pending_resources, pending_resources) Desktop.fail_release = True with pytest.raises(TaskCenterError) as caught: with BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) as runtime: with runtime.native_scope(tmp_path / 'operation'): pass runtime.result = 'verified completed task' assert caught.value.code == 'batch_resources_pending' assert caught.value.completed_result == 'verified completed task' assert pending_resources(tmp_path) with pytest.raises(TaskCenterError): guard_pending_resources(tmp_path) assert not list(tmp_path.rglob('uncertain.flag')) Desktop.fail_release = False assert cleanup_pending_resources(tmp_path) == () guard_pending_resources(tmp_path) assert len(Desktop.created) == 1 def test_cleanup_failure_takes_priority_over_cancel(tmp_path): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime Desktop.fail_release = True with pytest.raises(TaskCenterError) as caught: with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime: with runtime.native_scope(tmp_path / 'operation'): pass raise TaskCenterError('operation_cancelled', 'cancel requested') assert caught.value.code == 'batch_resources_pending' Desktop.fail_release = False from dx_productivity.trial_task_center.batch_runtime import cleanup_pending_resources cleanup_pending_resources(tmp_path) class Cad: created = [] def __init__(self, root, *, abort_event, desktop): self.root, self.desktop, self.abort_event = root, desktop, abort_event self.binding = SimpleNamespace(pid=500, guid='owned') self.closed = False def __enter__(self): self.desktop.__enter__() self.created.append(self) return self def checkpoint(self, evidence): assert not self.closed def start_worker(self, *args, **kwargs): raise AssertionError('monitor is injected') def __exit__(self, *_exc): self.closed = True self.desktop.close() def command_for(tmp_path, name='one'): root = tmp_path / 'data' / name root.mkdir(parents=True) project = tmp_path / ('project-' + name) / 'test.prj' project.parent.mkdir() project.write_text('project identity') (root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\n' f'task\tjob\t{project}\told\tnew\n') return ('cscript', '//nologo', 'worker.vbs', str(root), 'python', 'job'), project def worker_runtime(tmp_path, monkeypatch, monitor): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime from dx_productivity.trial_task_center import worker_monitor monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor) Cad.created = [] return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad) def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, monkeypatch): import subprocess from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released one, _ = command_for(tmp_path) two, _ = command_for(tmp_path, 'two') runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k: subprocess.CompletedProcess(command, 0, '', '')) with runtime: runtime.run_worker(one, 5) assert project_checkpoint_released(Path(one[3])) runtime.run_worker(two, 5) assert project_checkpoint_released(Path(two[3])) assert len(Cad.created) == len(Desktop.created) == 1 assert Cad.created[0].closed and Desktop.created[0].released @pytest.mark.parametrize('second_fails', [False, True]) def test_prewrite_failure_releases_old_desktop_and_retries_only_once(tmp_path, monkeypatch, second_fails): import subprocess from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError command, _ = command_for(tmp_path) calls = [] def monitor(command, *args, **kwargs): calls.append(len(calls)) if len(calls) == 2: assert Desktop.created[0].released return subprocess.CompletedProcess(command, 2 if len(calls) == 1 or second_fails else 0, '', 'startup') runtime = worker_runtime(tmp_path, monkeypatch, monitor) with runtime: if second_fails: with pytest.raises(WorkerMonitorError): runtime.run_worker(command, 5) else: runtime.run_worker(command, 5) assert len(calls) == len(Desktop.created) == len(Cad.created) == 2 assert runtime.generation == 2 releases = json.loads(runtime.path.read_text())['generation_releases'] assert [release['generation'] for release in releases] == [1, 2] assert [release['desktop'] for release in releases] == [desktop.name for desktop in Desktop.created] assert all(release['cad_pid'] == 500 and release['cad_guid'] == 'owned' and release['cleanup']['desktop_confirmation']['released'] is True and release['released_at'] > 0 for release in releases) def test_possible_mutation_prevents_automatic_replay(tmp_path, monkeypatch): from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError command, _ = command_for(tmp_path) calls = [] def monitor(*args, **kwargs): calls.append(1) path = next((runtime.path.parent / 'attempts').rglob('job.json')) record = json.loads(path.read_text()) record['state'] = 'mutation_possible' path.write_text(json.dumps(record)) raise WorkerMonitorError('lost connection after possible write') runtime = worker_runtime(tmp_path, monkeypatch, monitor) with runtime, pytest.raises(WorkerMonitorError): runtime.run_worker(command, 5) assert len(calls) == len(Desktop.created) == 1 def test_previous_checkpoint_invalidated_before_new_worker(tmp_path, monkeypatch): import subprocess from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError command, _ = command_for(tmp_path) calls = [] def monitor(*args, **kwargs): assert not project_checkpoint_released(Path(command[3])) calls.append(1) return subprocess.CompletedProcess(command, 0, '', '') runtime = worker_runtime(tmp_path, monkeypatch, monitor) with runtime: runtime.run_worker(command, 5) assert project_checkpoint_released(Path(command[3])) def fail_checkpoint(*args): raise WorkerMonitorError('cannot certify empty project') runtime.cad.checkpoint = fail_checkpoint # No retry after evidence has been marked as possibly modified. path = next((runtime.path.parent / 'attempts').rglob('job.json')) record = json.loads(path.read_text()) record['state'] = 'mutation_possible' path.write_text(json.dumps(record)) with pytest.raises(WorkerMonitorError): runtime.run_worker(command, 5) assert not project_checkpoint_released(Path(command[3])) def test_attempt_publication_failure_never_launches_worker(tmp_path, monkeypatch): from dx_productivity.trial_task_center import batch_attempt command, _ = command_for(tmp_path) runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: pytest.fail('must not launch')) def fail(*args, **kwargs): raise OSError('disk unavailable') monkeypatch.setattr(batch_attempt, 'prepare_attempt', fail) with runtime, pytest.raises(OSError): runtime.run_worker(command, 5) assert not Cad.created assert not Desktop.created from dx_productivity.trial_task_center.batch_resources import pending_resources assert pending_resources(runtime.root) == () def test_boundary_pause_does_not_abort_current_cad_startup(tmp_path, monkeypatch): import subprocess from threading import Event command, _ = command_for(tmp_path) runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k: subprocess.CompletedProcess(command, 0, '', '')) pause, worker_abort = Event(), Event() runtime.stop_event = pause original_enter = Cad.__enter__ def enter(cad): pause.set() # User requests pause after the current task has begun. assert cad.abort_event is worker_abort assert not cad.abort_event.is_set() return original_enter(cad) monkeypatch.setattr(Cad, '__enter__', enter) with runtime: runtime.run_worker(command, 5, abort_event=worker_abort) assert not list(tmp_path.rglob('uncertain.flag')) def test_cleanup_still_runs_when_resource_journal_write_fails(tmp_path, monkeypatch): from dx_productivity.trial_task_center import batch_runtime runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) with pytest.raises(TaskCenterError) as caught: with runtime: with runtime.native_scope(tmp_path / 'read'): pass real_write = batch_runtime.write_record def failing_write(path, record): if record['state'] == 'closing': raise OSError('journal unavailable') return real_write(path, record) monkeypatch.setattr(batch_runtime, 'write_record', failing_write) assert Desktop.created[0].released assert caught.value.code == 'batch_resources_pending' monkeypatch.setattr(batch_runtime, 'write_record', real_write) runtime.close() def test_real_startup_failure_with_released_lease_can_retry(tmp_path, monkeypatch): import io import subprocess from dx_productivity.trial_task_center import isolated_cad, worker_monitor from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime command, _ = command_for(tmp_path) starts = [] def executable(): starts.append(1) if len(starts) == 1: raise worker_monitor.WorkerMonitorError('registry temporarily unavailable') return 'cad.exe' class BootstrapDesktop(Desktop): def start_process(self, command, **options): process = super().start_process(command, **options) process.stderr = io.BytesIO() process.wait = lambda timeout: process.returncode if command[0] == 'cad.exe': self.cad = process process.returncode = None else: action, evidence, pid = command environment = options['env'] binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned', 'token': environment['DXPART_SESSION_TOKEN']} if action == 'probe': Path(evidence).write_text(json.dumps(binding)) elif action == 'checkpoint': Path(evidence).write_text(json.dumps({'binding': binding, 'project_path': '', 'project_closed': True})) elif action == 'quit': self.cad.returncode = 0 return process monkeypatch.setattr(isolated_cad, '_designer_executable', executable) monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args) monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda *a, **k: subprocess.CompletedProcess(command, 0, '', '')) with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=BootstrapDesktop) as runtime: runtime.run_worker(command, 5) assert len(starts) == len(Desktop.created) == 2 assert all(desktop.released for desktop in Desktop.created) def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path): import hashlib from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime command, project = command_for(tmp_path) database = project.parent / 'database.dat' database.write_bytes(b'database') evidence = Path(command[3]) / 'native' evidence.mkdir() proof = {'version': 1, 'native_read_only': True, 'workers_closed': True, 'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()), 'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest() for path in (project, database)}} path = evidence / 'native-retry-proof.json' path.write_text(json.dumps(proof)) with BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) as runtime: with runtime.native_scope(evidence) as scope: pass proof['binding'] = scope.identity path.write_text(json.dumps(proof)) database.write_bytes(b'changed externally') assert not runtime.recover_native_failure(project, evidence, 'timeout') database.write_bytes(b'database') assert runtime.recover_native_failure(project, evidence, 'timeout') assert Desktop.created[0].released from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError with pytest.raises(WorkerMonitorError, match='stale'): scope.start_process(('late-old-worker.exe',)) assert scope.identity['generation'] == 1 and runtime.generation == 2 scope.close() # A late close cannot grant the old operation a new identity. assert str(evidence.resolve()) not in runtime.record['native_operations'] assert not runtime.recover_native_failure(project, evidence, 'another timeout') assert len(Desktop.created) == 2 def test_mock_native_read_recovery_spends_same_project_worker_retry_budget(tmp_path, monkeypatch): """Real durable proof/runtime logic; desktop, CAD and worker failure are fakes.""" import hashlib from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError command, project = command_for(tmp_path) database = project.parent / 'database.dat' database.write_bytes(b'unchanged database') evidence = Path(command[3]) / 'native' evidence.mkdir() calls = [] def fail_worker(*args, **kwargs): calls.append(1) raise WorkerMonitorError('prewrite failure after native recovery') runtime = worker_runtime(tmp_path, monkeypatch, fail_worker) with runtime: with runtime.native_scope(evidence) as scope: pass proof = {'version': 1, 'native_read_only': True, 'workers_closed': True, 'binding': scope.identity, 'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()), 'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest() for path in (project, database)}} (evidence / 'native-retry-proof.json').write_text(json.dumps(proof)) assert runtime.recover_native_failure(project, evidence, 'read timeout') assert len(Desktop.created) == 2 and Desktop.created[0].released with pytest.raises(WorkerMonitorError, match='after native recovery'): runtime.run_worker(command, 5) assert calls == [1] assert len(Desktop.created) == 2 assert not runtime.recover_native_failure(project, evidence, 'another read timeout') assert all(desktop.released for desktop in Desktop.created) assert runtime.record['retries'][str(project.resolve()).casefold()]['count'] == 1 def test_mock_different_projects_each_get_one_retry_without_accumulating_budget(tmp_path, monkeypatch): """Real per-project accounting; fake CAD/desktop and unchanged prewrite failures.""" import subprocess from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError one, project_one = command_for(tmp_path) two, project_two = command_for(tmp_path, 'two') calls = {one[3]: 0, two[3]: 0} def monitor(command, *args, **kwargs): calls[command[3]] += 1 if calls[command[3]] == 2: return subprocess.CompletedProcess(command, 0, '', '') raise WorkerMonitorError('unchanged prewrite failure') runtime = worker_runtime(tmp_path, monkeypatch, monitor) with runtime: runtime.run_worker(one, 5) assert len(Desktop.created) == 2 runtime.run_worker(two, 5) assert len(Desktop.created) == 3 # Each project gets its own single rebuild. for command in (one, two): with pytest.raises(WorkerMonitorError, match='prewrite failure'): runtime.run_worker(command, 5) assert len(Desktop.created) == 3 # Success cannot replenish either budget. assert calls == {one[3]: 3, two[3]: 3} assert set(runtime.record['retries']) == { str(project_one.resolve()).casefold(), str(project_two.resolve()).casefold()} assert all(value['count'] == 1 for value in runtime.record['retries'].values()) assert all(desktop.released for desktop in Desktop.created) def test_release_publication_failure_can_be_reconciled_without_replaying(tmp_path, monkeypatch): from dx_productivity.trial_task_center import batch_runtime runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) real_write = batch_runtime.write_record def failing_write(path, record): if record['state'] == 'released': raise OSError('cannot publish release') return real_write(path, record) with pytest.raises(TaskCenterError): with runtime: with runtime.native_scope(tmp_path / 'read'): pass monkeypatch.setattr(batch_runtime, 'write_record', failing_write) assert Desktop.created[0].released and batch_runtime.pending_resources(tmp_path) monkeypatch.setattr(batch_runtime, 'write_record', real_write) assert batch_runtime.cleanup_pending_resources(tmp_path) == () assert len(Desktop.created) == 1 releases = json.loads(runtime.path.read_text())['generation_releases'] assert len(releases) == 1 and releases[0]['generation'] == 1 @pytest.mark.parametrize('record', [ {'version': 1, 'session_id': 'old', 'state': 'blocked'}, {'version': 1, 'session_id': 'old', 'state': 'released'}, [], ]) def test_unknown_resource_evidence_never_authorizes_cleanup(tmp_path, record): from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources, pending_resources path = tmp_path / 'batch-resources' / 'old' / 'session.json' path.parent.mkdir(parents=True) path.write_text(json.dumps(record)) assert pending_resources(tmp_path) assert cleanup_pending_resources(tmp_path) assert json.loads(path.read_text()) == record def test_stale_generation_and_unknown_checkpoint_schema_are_rejected(tmp_path, monkeypatch): import subprocess from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released command, _ = command_for(tmp_path) runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: subprocess.CompletedProcess(command, 0, '', '')) with runtime: runtime.run_worker(command, 5) path = Path(command[3]) / 'project-checkpoint.json' checkpoint = json.loads(path.read_text()) assert project_checkpoint_released(Path(command[3])) for changes in ({'version': 999}, {'generation': 999}, {'attempt_id': 'old'}, {'cad_guid': 'foreign'}): path.write_text(json.dumps(dict(checkpoint, **changes))) assert not project_checkpoint_released(Path(command[3])) def test_restart_reconciles_only_confirmed_disappearance_without_replacement(tmp_path, monkeypatch): from dx_productivity.trial_task_center import batch_resources, isolated_desktop from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime Desktop.fail_release = True runtime = BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) with pytest.raises(TaskCenterError): with runtime: with runtime.native_scope(tmp_path / 'read'): pass # Simulate restarting the UI: there are no retained owned handles to act on. batch_resources.LIVE_RESOURCES.pop(str(runtime.path)) present = [True] class Api: def confirm_desktop_released(self, name): assert name == runtime.record['desktop'] if present[0]: raise OSError('desktop still held by external thread') monkeypatch.setattr(isolated_desktop, '_Win32Api', Api) assert batch_resources.cleanup_pending_resources(tmp_path) present[0] = False assert batch_resources.cleanup_pending_resources(tmp_path) == () assert len(Desktop.created) == 1 @pytest.mark.parametrize('conflict', [ 'bootstrap_binding', 'bootstrap_json', 'bootstrap_schema', 'helper_identity', 'checkpoint_binding', 'checkpoint_json', 'worker_identity', ]) def test_identity_conflict_never_rebuilds_or_launches_another_worker(tmp_path, monkeypatch, conflict): import io import subprocess from dx_productivity.trial_task_center import isolated_cad, worker_monitor from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime command, _ = command_for(tmp_path) workers = [] class IdentityDesktop(Desktop): def start_process(self, launch, **options): process = super().start_process(launch, **options) process.stderr = io.BytesIO(b'CAD COM GUID does not match its owned session') process.wait = lambda timeout: process.returncode if launch[0] == 'cad.exe': self.cad = process process.returncode = None return process action, evidence, pid = launch binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned', 'token': options['env']['DXPART_SESSION_TOKEN']} if action == 'probe': if conflict == 'helper_identity': process.returncode = 4 # Private helper identity-conflict exit. return process if conflict == 'bootstrap_binding': binding['pid'] = pid + 1 if conflict == 'bootstrap_schema': binding.pop('token') Path(evidence).write_text('{' if conflict == 'bootstrap_json' else json.dumps(binding)) elif action == 'checkpoint': closing = Path(evidence).name.startswith('cad-closed-') if closing and conflict == 'checkpoint_binding': binding['guid'] = 'foreign' record = {'binding': binding, 'project_path': '', 'project_closed': True} Path(evidence).write_text('{' if closing and conflict == 'checkpoint_json' else json.dumps(record)) elif action == 'quit': self.cad.returncode = 0 return process monkeypatch.setattr(isolated_cad, '_designer_executable', lambda: 'cad.exe') monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args) def monitor(*args, **kwargs): workers.append(1) if conflict == 'worker_identity': return subprocess.CompletedProcess(command, 2, '', 'private CAD GUID mismatch; no project was changed') return subprocess.CompletedProcess(command, 0, '', '') monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor) runtime = BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=IdentityDesktop) with runtime, pytest.raises(worker_monitor.WorkerMonitorError): runtime.run_worker(command, 5) assert len(Desktop.created) == 1 assert len(workers) == (1 if conflict.startswith('checkpoint') or conflict == 'worker_identity' else 0) assert runtime.generation == 1 and not runtime.record['retries'] assert Desktop.created[0].released def test_request_session_identity_is_uuid_bound(tmp_path): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime session = '12345678-1234-5678-1234-567812345678' runtime = BatchCadRuntime(tmp_path, 'bom_import', session_id=session) assert runtime.session_id == session.replace('-', '') with pytest.raises(ValueError): BatchCadRuntime(tmp_path, 'bom_import', session_id='../escape') def test_cleanup_preserves_primary_bom_error_and_report(tmp_path): from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime primary = TaskCenterError('bom_inspection', 'read failed') primary.bom_report = object() Desktop.fail_release = True with pytest.raises(TaskCenterError) as caught: with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime: with runtime.native_scope(tmp_path / 'operation'): pass raise primary error = caught.value assert error.primary_error is primary assert error.cleanup_error is not None assert error.bom_report is primary.bom_report assert 'BOM' in error.message and '已核验任务结果保留' not in error.message assert runtime.record['primary_error']['code'] == primary.code assert runtime.record['cleanup_error'] Desktop.fail_release = False from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources cleanup_pending_resources(tmp_path) def test_attempts_live_under_short_session_root(tmp_path, monkeypatch): import subprocess command, _ = command_for(tmp_path) runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k: subprocess.CompletedProcess(command, 0, '', '')) with runtime: runtime.run_worker(command, 5) assert list((runtime.path.parent / 'attempts').rglob('job.json')) assert not (Path(command[3]) / 'batch-attempts').exists()