From a94276494a5cd5cffe79dce7d336c87f6ced5dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Wed, 21 Oct 2026 14:49:14 +0800 Subject: [PATCH] fix: guard standalone CAD entries and preserve cleanup diagnostics --- .../trial_task_center/bom_batch.py | 4 +- .../trial_task_center/bom_import_report.py | 8 ++- .../trial_task_center/bom_import_state.py | 6 +- .../trial_task_center/inventory.py | 9 ++- .../trial_task_center/native_inventory.py | 65 ++++++++++++++--- .../trial_task_center/quick_run.py | 3 +- .../trial_task_center/recovery.py | 8 ++- .../trial_task_center/service.py | 60 +++++++++++----- .../trial_task_center/single_step.py | 5 +- tests/v1_1/test_batch_runtime_integration.py | 23 ++++++ tests/v1_1/test_bom_import_state.py | 13 ++++ tests/v1_1/test_native_inventory_reader.py | 72 +++++++++++++++++++ tests/v1_1/test_service_plan_safety.py | 30 ++++++++ 13 files changed, 269 insertions(+), 37 deletions(-) diff --git a/src/dx_productivity/trial_task_center/bom_batch.py b/src/dx_productivity/trial_task_center/bom_batch.py index a42dd87..f506811 100644 --- a/src/dx_productivity/trial_task_center/bom_batch.py +++ b/src/dx_productivity/trial_task_center/bom_batch.py @@ -55,11 +55,11 @@ def _assert_editable(service, task_id): raise TaskCenterError('target_plan_locked', f'工程任务当前不能修改 BOM 计划:{context.task.name}') -def preview_batch(service, task_id, path, *, stop_event=None, on_progress=None): +def preview_batch(service, task_id, path, *, stop_event=None, on_progress=None, runtime=None): """Strict adapter to the same validation engine used by the import report.""" from .bom_import_report import inspect_batch - report = inspect_batch(service, task_id, path, stop_event=stop_event, on_progress=on_progress) + report = inspect_batch(service, task_id, path, stop_event=stop_event, on_progress=on_progress, runtime=runtime) if report.preview is not None: return report.preview failed = next((row for row in report.rows if row.status == 'failed'), None) diff --git a/src/dx_productivity/trial_task_center/bom_import_report.py b/src/dx_productivity/trial_task_center/bom_import_report.py index c8db7c5..b5bd896 100644 --- a/src/dx_productivity/trial_task_center/bom_import_report.py +++ b/src/dx_productivity/trial_task_center/bom_import_report.py @@ -134,8 +134,12 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, report = None try: with BatchCadRuntime(service._logging.root, 'bom_import', stop_event) as owned_runtime: - report = inspect_batch(service, task_id, path, stop_event=stop_event, - on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime) + try: + report = inspect_batch(service, task_id, path, stop_event=stop_event, + on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime) + except BomInspectionError as error: + report = error.bom_report + raise owned_runtime.result = report return report except BatchResourceCleanupError as error: diff --git a/src/dx_productivity/trial_task_center/bom_import_state.py b/src/dx_productivity/trial_task_center/bom_import_state.py index 6f9f434..f55c143 100644 --- a/src/dx_productivity/trial_task_center/bom_import_state.py +++ b/src/dx_productivity/trial_task_center/bom_import_state.py @@ -90,9 +90,11 @@ class BomImportMachine: return False cancelled_error = isinstance(error, TaskCenterError) and error.code == 'operation_cancelled' if error is not None and not cancelled_error: - from .bom_import_report import BomInspectionError + from .bom_import_report import BomImportReport reason = error.message if isinstance(error, TaskCenterError) else str(error) - report = error.bom_report if isinstance(error, BomInspectionError) else None + report = getattr(error, 'bom_report', None) + if not isinstance(report, BomImportReport): + report = None self.state = replace(self.state, phase=BomImportPhase.FAILED, report=report, message=f'本次 BOM 导入未完成:{reason[:240]};原有预览和编辑保留。') elif cancelled_error or self.state.phase is BomImportPhase.CANCELLING: diff --git a/src/dx_productivity/trial_task_center/inventory.py b/src/dx_productivity/trial_task_center/inventory.py index dedb137..67dfdbb 100644 --- a/src/dx_productivity/trial_task_center/inventory.py +++ b/src/dx_productivity/trial_task_center/inventory.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect from pathlib import Path from .errors import TaskCenterError @@ -58,11 +59,15 @@ def read_component_inventory(path: Path, project_path: Path, def collect_target_inventory(preparer, project_path: Path, - old_part_numbers: tuple[str, ...], *, stop_event=None, on_progress=None) -> tuple[ComponentTarget, ...]: + old_part_numbers: tuple[str, ...], *, stop_event=None, on_progress=None, runtime=None) -> tuple[ComponentTarget, ...]: if not old_part_numbers: raise TaskCenterError("target_inventory", "at least one old Part Number is required") from .operation_control import check_cancelled, control_options check_cancelled(stop_event) - rows = preparer.collect_inventory(project_path, old_part_numbers, **control_options(stop_event, on_progress)) + runtime_options = ({'runtime': runtime} if runtime is not None and any( + parameter.name == 'runtime' or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in inspect.signature(preparer.collect_inventory).parameters.values()) else {}) + rows = preparer.collect_inventory(project_path, old_part_numbers, **runtime_options, + **control_options(stop_event, on_progress)) check_cancelled(stop_event) return rows diff --git a/src/dx_productivity/trial_task_center/native_inventory.py b/src/dx_productivity/trial_task_center/native_inventory.py index 7fb6e4e..9e24919 100644 --- a/src/dx_productivity/trial_task_center/native_inventory.py +++ b/src/dx_productivity/trial_task_center/native_inventory.py @@ -17,6 +17,15 @@ from .native_inventory_protocol import NativeInventoryUnsupported, read_ascii_di from .operation_control import check_cancelled, report_progress +_RETRY_NATIVE_READ = object() +_NATIVE_CRASH_CODES = frozenset({0xC0000005, 0xC000001D, 0xC0000094, + 0xC00000FD, 0xC0000374, 0xC0000409}) + + +class _NativeInventoryProcessError(NativeInventoryUnsupported): + """A launch, watchdog or explicit crash failure, separate from invalid input.""" + + def _hash(path): with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() @@ -80,10 +89,15 @@ class NativeInventoryReader: except (NativeInventoryUnsupported, OSError, ImportError, UnicodeError, TaskCenterError) as error: report_progress(on_progress, f'使用 CAD 核验位号:{str(error)[:160]}') return None - return self._read(project, text, block, snapshot, database, tools, old_part_numbers, - stop_event=stop_event, on_progress=on_progress, runtime=runtime) + for attempt in range(2): + result = self._read(project, text, block, snapshot, database, tools, old_part_numbers, + stop_event=stop_event, on_progress=on_progress, runtime=runtime, + allow_retry=runtime is not None and attempt == 0) + if result is not _RETRY_NATIVE_READ: + return result + return None - def _read(self, project, text, block, snapshot, database, tools, old_part_numbers, *, stop_event, on_progress, runtime=None): + def _read(self, project, text, block, snapshot, database, tools, old_part_numbers, *, stop_event, on_progress, runtime=None, allow_retry=False): token = uuid4().hex evidence = self.data_root / 'quick-run-history' / ('native-inventory-' + token) evidence.mkdir(parents=True, exist_ok=False) @@ -152,9 +166,31 @@ class NativeInventoryReader: if record['state'] == 'uncertain': raise TaskCenterError('inventory_cleanup', f'原生采集清理状态不确定:{evidence}') from error _verify_sources(source_hashes) - record.update(state='fallback', reason=str(error)) - report_progress(on_progress, f'原生清单未能完整核对,改用 CAD 采集:{str(error)[:160]}') + retry = False + cleanup = getattr(desktop, 'cleanup_record', None) + if (allow_retry and isinstance(error, _NativeInventoryProcessError) + and clean and isinstance(cleanup, dict) and cleanup.get('workers_closed') is True): + check_cancelled(stop_event) + from .batch_resources import write_record + write_record(evidence / 'native-retry-proof.json', { + 'version': 1, 'native_read_only': True, 'project_path': str(project), + 'source_hashes': {str(path): digest for path, digest in source_hashes.items()}, + 'workers_closed': True, 'evidence_root': str(evidence.resolve()), + }) + _verify_sources(source_hashes) + retry = runtime.recover_native_failure(project, evidence, str(error)) + record.update(state='retrying' if retry else 'fallback', reason=str(error)) + if retry: + result = _RETRY_NATIVE_READ + report_progress(on_progress, '原生读取会话已安全释放,正在重新核验一次…') + else: + report_progress(on_progress, f'原生清单未能完整核对,改用 CAD 采集:{str(error)[:160]}') except TaskCenterError as error: + if record['state'] == 'uncertain': + try: + (evidence / 'uncertain.flag').write_text(error.message, encoding='utf-8') + except OSError as marker_error: + record['uncertain_marker_error'] = str(marker_error) if record['state'] != 'uncertain': record.update(state='cancelled' if error.code == 'operation_cancelled' else 'failed', reason=error.message) raise @@ -182,19 +218,32 @@ class NativeInventoryReader: started = time.monotonic() metric = {'name': name, 'report': report_name, 'exit_code': None} record['tools'].append(metric) + process = None try: with (output / (log_name + '.stdout.txt')).open('wb') as stdout, ( output / (log_name + '.stderr.txt')).open('wb') as stderr: - process = desktop.start_process(command, cwd=workspace, env=environment, - stdout=stdout, stderr=stderr) + try: + process = desktop.start_process(command, cwd=workspace, env=environment, + stdout=stdout, stderr=stderr) + except OSError as error: + raise _NativeInventoryProcessError(f'{name} 启动失败:{error}') from error deadline = started + self.timeout while process.poll() is None: check_cancelled(stop_event) if time.monotonic() >= deadline: - raise NativeInventoryUnsupported(f'{name} 原生读取超时') + raise _NativeInventoryProcessError(f'{name} 原生读取超时') time.sleep(0.05) metric['exit_code'] = process.poll() + if metric['exit_code'] & 0xFFFFFFFF in _NATIVE_CRASH_CODES: + raise _NativeInventoryProcessError(f'{name} 原生读取进程崩溃:{metric["exit_code"]}') if metric['exit_code'] != 0: raise NativeInventoryUnsupported(f'{name} 返回退出码 {metric["exit_code"]}') finally: metric['seconds'] = time.monotonic() - started + if process is not None and callable(getattr(process, 'close', None)): + try: + process.close() + except Exception as error: + record.update(state='uncertain', cleanup_error=str(error)) + raise TaskCenterError('inventory_cleanup', + f'原生读取进程及其子进程未能确认退出:{error}') from error diff --git a/src/dx_productivity/trial_task_center/quick_run.py b/src/dx_productivity/trial_task_center/quick_run.py index 4c923fa..6c3cf57 100644 --- a/src/dx_productivity/trial_task_center/quick_run.py +++ b/src/dx_productivity/trial_task_center/quick_run.py @@ -338,7 +338,8 @@ class QuickRunPreparer: raise TaskCenterError("quick_run_uncertain", "a previous quick-run worker timed out or returned an uncorrelated status; reconcile the project before retrying") if self._process_runner is _run_process and not batch_mode: from .single_step import prepare_private_single_step - prepare_private_single_step(self, task_id, projects, rules, copy_part_number_to_value) + prepare_private_single_step(self, task_id, projects, rules, copy_part_number_to_value, + **({'runtime': runtime} if runtime is not None else {})) batch_mode = True current_root = self._data_root / "quick-run" if batch_session is None and self.evidence_path(task_id) == current_root and (current_root / "status.tsv").is_file(): diff --git a/src/dx_productivity/trial_task_center/recovery.py b/src/dx_productivity/trial_task_center/recovery.py index 58c43e7..c0df6ac 100644 --- a/src/dx_productivity/trial_task_center/recovery.py +++ b/src/dx_productivity/trial_task_center/recovery.py @@ -105,7 +105,7 @@ def require_current_cell_binding(snapshot, binding, project, metadata): return current -def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=False, confirmed=False): +def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=False, confirmed=False, runtime=None): if not confirmed: raise TaskCenterError('quick_run_recovery_confirmation', '核验恢复需要明确确认,不能作为自动重试执行。') maintain_quick_run_storage(preparer._data_root) @@ -227,7 +227,11 @@ def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=Fal launch = (str(preparer._cscript_resolver()), '//nologo', script_argument, root_argument, *_native_helper_launch_arguments(), job_id) try: - result = preparer._process_runner(launch, IDLE_PROGRESS_TIMEOUT_SECONDS) + from .quick_run import _run_process + if runtime is not None and preparer._process_runner is _run_process: + result = _run_process(launch, IDLE_PROGRESS_TIMEOUT_SECONDS, runtime=runtime) + else: + result = preparer._process_runner(launch, IDLE_PROGRESS_TIMEOUT_SECONDS) except (OSError, subprocess.SubprocessError) as error: raise TaskCenterError('quick_run_recovery', '恢复执行未能确认结束,原保护锁仍在;不要重新替换或删除证据。') from error (audit / 'worker-output.txt').write_text(result.stdout + '\n' + result.stderr, encoding='utf-8') diff --git a/src/dx_productivity/trial_task_center/service.py b/src/dx_productivity/trial_task_center/service.py index 2379d6c..ba9b51d 100644 --- a/src/dx_productivity/trial_task_center/service.py +++ b/src/dx_productivity/trial_task_center/service.py @@ -142,10 +142,11 @@ class TaskCenterService: return self._operation("export_bom_template", task_id, lambda: export_template(self._store, task_id, path)) - def preview_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None): + def preview_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, runtime=None): from .bom_batch import preview_batch return self._idle_quick_run_operation('preview_bom_batch', task_id, - lambda: preview_batch(self, task_id, path, **control_options(stop_event, on_progress))) + lambda: preview_batch(self, task_id, path, **control_options(stop_event, on_progress), + **({'runtime': runtime} if runtime is not None else {}))) def save_bom_batch(self, preview, rows_by_task): from .bom_batch import save_batch @@ -388,10 +389,15 @@ class TaskCenterService: return self._operation("list_projects", task_id, lambda: self._store.list_projects(task_id)) def collect_target_inventory(self, task_id: str, old_part_numbers: tuple[str, ...] | None = None, - *, stop_event=None, on_progress=None): + *, stop_event=None, on_progress=None, runtime=None): + if runtime is None: + return self._owned_cad_operation('collect_inventory', lambda owned: + self.collect_target_inventory(task_id, old_part_numbers, + stop_event=stop_event, on_progress=on_progress, runtime=owned), stop_event) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.collect_target_inventory(task_id, old_part_numbers, **control_options(stop_event, on_progress)) + return delegated.collect_target_inventory(task_id, old_part_numbers, runtime=runtime, + **control_options(stop_event, on_progress)) def collect(): from .bom_batch import _assert_editable _assert_editable(self, task_id) @@ -409,7 +415,7 @@ class TaskCenterService: project_path=str(project[0].path)) as metric: report_progress(on_progress, f'正在采集工程:{project[0].display_name}') rows = collect_target_inventory(self._quick_run_preparer, project[0].path, parts, - **control_options(stop_event, on_progress)) + runtime=runtime, **control_options(stop_event, on_progress)) check_cancelled(stop_event) metric['completed_units'] = len(rows) with self._store.transaction(): @@ -555,11 +561,14 @@ class TaskCenterService: return self._operation("expected_probe_path", task_id, expected_path) - def prepare_quick_run(self, task_id: str) -> QuickRunResult: + def prepare_quick_run(self, task_id: str, *, runtime=None) -> QuickRunResult: """Create the legacy VBS handoff without starting a task-center trial.""" + if runtime is None: + return self._owned_cad_operation('prepare_quick_run', lambda owned: + self.prepare_quick_run(task_id, runtime=owned)) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.prepare_quick_run(task_id) + return delegated.prepare_quick_run(task_id, runtime=runtime) def prepare() -> QuickRunResult: task = self._store.get_task(task_id) preparer = (self._quick_run_preparer.ensure_batch_handoff @@ -569,6 +578,7 @@ class TaskCenterService: self._store.list_projects(task_id), self._execution_rules(task_id), copy_part_number_to_value=task.copy_part_number_to_value, + **({'runtime': runtime} if task.target_plan_mode else {}), ) append_task_event( self._logging, @@ -624,11 +634,14 @@ class TaskCenterService: with _quick_run_lease(self._logging.root): return self._operation("set_value_policy", task_id, update) - def execute_quick_run(self, task_id: str, *, expected_item: tuple[str, str, str, str] | None = None) -> QuickRunStepResult: + def execute_quick_run(self, task_id: str, *, expected_item: tuple[str, str, str, str] | None = None, runtime=None) -> QuickRunStepResult: """Execute one replacement rule through restoration and verification.""" + if runtime is None: + return self._owned_cad_operation('quick_run_step', lambda owned: + self.execute_quick_run(task_id, expected_item=expected_item, runtime=owned)) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.execute_quick_run(task_id, expected_item=expected_item) + return delegated.execute_quick_run(task_id, expected_item=expected_item, runtime=runtime) def execute() -> QuickRunStepResult: task = self._store.get_task(task_id) result = self._quick_run_preparer.execute_step( @@ -637,6 +650,7 @@ class TaskCenterService: self._execution_rules(task_id), copy_part_number_to_value=task.copy_part_number_to_value, **({"expected_item": expected_item} if expected_item is not None else {}), + runtime=runtime, ) self._synchronize_quick_run_progress(task_id) append_task_event( @@ -718,11 +732,14 @@ class TaskCenterService: finally: self._batch_feedback = None - def prepare_failed_quick_run(self, task_id: str) -> QuickRunProgress: + def prepare_failed_quick_run(self, task_id: str, *, runtime=None) -> QuickRunProgress: """Reset only failed projects and discard the finished queue handoff.""" + if runtime is None: + return self._owned_cad_operation('prepare_failed_quick_run', lambda owned: + self.prepare_failed_quick_run(task_id, runtime=owned)) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.prepare_failed_quick_run(task_id) + return delegated.prepare_failed_quick_run(task_id, runtime=runtime) def prepare_retry() -> QuickRunProgress: task = self._store.get_task(task_id) if self._quick_run_preparer.has_interrupted(task_id): @@ -768,6 +785,7 @@ class TaskCenterService: self._store.list_projects(task_id), self._execution_rules(task_id), copy_part_number_to_value=task.copy_part_number_to_value, + runtime=runtime, ) return self._quick_run_preparer.progress( task_id, @@ -787,6 +805,13 @@ class TaskCenterService: return self._idle_quick_run_operation('start_new_round', task_id, lambda: start_new_round(self, task_id)) + def _owned_cad_operation(self, kind, action, stop_event=None): + from .batch_runtime import BatchCadRuntime + with BatchCadRuntime(self._logging.root, kind, stop_event) as runtime: + result = action(runtime) + runtime.result = result + return result + def pending_batch_resources(self): from .batch_runtime import pending_resources return pending_resources(self._logging.root) @@ -800,10 +825,10 @@ class TaskCenterService: return self._operation('cleanup_batch_resources', None, lambda: cleanup_pending_resources(self._logging.root)) - def inspect_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, on_phase=None): + def inspect_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, on_phase=None, runtime=None): from .bom_import_report import inspect_batch return self._idle_quick_run_operation('inspect_bom_batch', task_id, - lambda: inspect_batch(self, task_id, path, on_phase=on_phase, + lambda: inspect_batch(self, task_id, path, on_phase=on_phase, runtime=runtime, **control_options(stop_event, on_progress))) def _pause_quick_run_queue(self, task_id: str, message: str) -> None: @@ -860,10 +885,13 @@ class TaskCenterService: return delegated.has_interrupted_quick_run(task_id) return self._quick_run_preparer.has_interrupted(task_id) - def recover_quick_run(self, task_id: str, *, confirmed: bool = False) -> QuickRunStepResult: + def recover_quick_run(self, task_id: str, *, confirmed: bool = False, runtime=None) -> QuickRunStepResult: + if runtime is None: + return self._owned_cad_operation('recover_quick_run', lambda owned: + self.recover_quick_run(task_id, confirmed=confirmed, runtime=owned)) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.recover_quick_run(task_id, confirmed=confirmed) + return delegated.recover_quick_run(task_id, confirmed=confirmed, runtime=runtime) from .recovery import recover def reconcile() -> QuickRunStepResult: @@ -877,7 +905,7 @@ class TaskCenterService: raise TaskCenterError("quick_run_recovery", "工程包含手动执行阶段或失败记录,不能混用自动队列恢复。") result = recover(self._quick_run_preparer, task_id, projects, self._execution_rules(task_id), copy_part_number_to_value=task.copy_part_number_to_value, - confirmed=confirmed) + confirmed=confirmed, runtime=runtime) if task.status in {TaskStatus.PAUSED, TaskStatus.FAILED}: self._store.set_task_status(task_id, TaskStatus.READY, summary="interrupted quick-run reconciled") self._synchronize_quick_run_progress(task_id) diff --git a/src/dx_productivity/trial_task_center/single_step.py b/src/dx_productivity/trial_task_center/single_step.py index 3c4ed9a..1e49811 100644 --- a/src/dx_productivity/trial_task_center/single_step.py +++ b/src/dx_productivity/trial_task_center/single_step.py @@ -5,7 +5,7 @@ from .errors import TaskCenterError from .schematic_backup import create_schematic_backup -def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_number_to_value): +def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_number_to_value, *, runtime=None): progress = preparer.progress(task_id, projects, rules, copy_part_number_to_value=copy_part_number_to_value) if progress.queue_complete or progress.next_project_path is None: @@ -15,4 +15,5 @@ def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_nu create_schematic_backup(preparer.data_root, progress.next_project_path, preparer.data_root / 'quick-run-history' / ('batch-' + uuid4().hex)) preparer.ensure_batch_handoff(task_id, projects, rules, - copy_part_number_to_value=copy_part_number_to_value) + copy_part_number_to_value=copy_part_number_to_value, + **({'runtime': runtime} if runtime is not None else {})) diff --git a/tests/v1_1/test_batch_runtime_integration.py b/tests/v1_1/test_batch_runtime_integration.py index 6ab590c..3a5fc10 100644 --- a/tests/v1_1/test_batch_runtime_integration.py +++ b/tests/v1_1/test_batch_runtime_integration.py @@ -8,6 +8,7 @@ import pytest from dx_productivity.trial_task_center import task_queue from dx_productivity.trial_task_center.errors import TaskCenterError from test_task_queue import Service +from test_bom_batch import batch, records_for @pytest.fixture @@ -88,3 +89,25 @@ def test_reader_requires_current_batch_project_checkpoint(tmp_path, runtime_modu _require_released_reader(tmp_path) assert caught.value.code == 'inventory_cleanup' assert not isinstance(_read_failure(tmp_path, 'read failed'), ReleasedProjectReadError) + + +@pytest.mark.parametrize('read_error', [False, True]) +def test_bom_cleanup_failure_preserves_completed_or_partial_report(batch, tmp_path, runtime_module, read_error): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch + from dx_productivity.trial_task_center.bom_plan import write_bom_template + Runtime, scopes = runtime_module + Runtime.fail_cleanup = True + service, store, tasks, targets, calls = batch + if read_error: + def fail(*args, **kwargs): + raise TaskCenterError('inventory_cleanup', 'reader did not close') + service.for_project_task(tasks[0].task_id)._quick_run_preparer.collect_inventory = fail + with pytest.raises(TaskCenterError) as caught: + inspect_batch(service, tasks[0].task_id, + write_bom_template(tmp_path / 'cleanup.xlsx', records_for(tasks, targets))) + assert caught.value.code == 'batch_resources_pending' + report = caught.value.bom_report + assert report is not None and report.total_rows == 2 + assert report.failed_rows == (1 if read_error else 0) + assert report.passed_rows == (0 if read_error else 2) + assert len(scopes) == 1 and scopes[0].closed diff --git a/tests/v1_1/test_bom_import_state.py b/tests/v1_1/test_bom_import_state.py index 9a3d5a4..8cd3e8f 100644 --- a/tests/v1_1/test_bom_import_state.py +++ b/tests/v1_1/test_bom_import_state.py @@ -76,3 +76,16 @@ def test_failed_import_preserves_attached_row_report_and_retry_resets_it(): assert machine.state.phase is Phase.FAILED and machine.state.report is report machine.start() assert machine.state.report is None and machine.state.phase is Phase.READING + + +def test_resource_cleanup_failure_preserves_bom_row_report_after_cancellation(): + from dx_productivity.trial_task_center.batch_runtime import BatchResourceCleanupError + machine = BomImportMachine() + attempt = machine.start() + machine.cancel(attempt) + report = BomImportReport((), (), errors=('project read failed',)) + error = BatchResourceCleanupError('desktop cleanup failed') + error.bom_report = report + machine.finish(attempt, error=error) + assert machine.state.phase is Phase.FAILED and machine.state.report is report + assert 'desktop cleanup failed' in machine.state.message diff --git a/tests/v1_1/test_native_inventory_reader.py b/tests/v1_1/test_native_inventory_reader.py index b612cef..a2a071c 100644 --- a/tests/v1_1/test_native_inventory_reader.py +++ b/tests/v1_1/test_native_inventory_reader.py @@ -90,6 +90,78 @@ def test_native_reader_borrows_batch_desktop_and_closes_only_native_scope(tmp_pa assert len(evidence) == 1 and evidence[0].parent.name == 'quick-run-history' +@pytest.mark.parametrize('fail_close', [False, True]) +def test_native_jobs_release_descendants_before_next_command_or_fallback(tmp_path, fail_close): + project, reader, stop, calls, released = reader_fixture(tmp_path) + factory = reader.desktop_factory + events = [] + def desktop(*args, **options): + value = factory(*args, **options) + original = value.start_process + def start(command, **kwargs): + events.append('start') + process = original(command, **kwargs) + def close(): + events.append('close') + if fail_close: + raise OSError('native descendant remained alive') + process.close = close + return process + value.start_process = start + return value + reader.desktop_factory = desktop + if fail_close: + with pytest.raises(TaskCenterError) as caught: + reader.read(project, ('OLD',)) + assert caught.value.code == 'inventory_cleanup' + assert events == ['start', 'close'] + assert list((tmp_path / 'evidence').rglob('uncertain.flag')) + else: + assert reader.read(project, ('OLD',)).components + assert events == ['start', 'close', 'start', 'close'] + + +@pytest.mark.parametrize('failure', ['launch', 'timeout', 'crash', 'input_error', 'parser']) +def test_only_transient_native_process_failure_requests_one_proven_rebuild(tmp_path, failure): + project, reader, stop, calls, released = reader_fixture(tmp_path, corrupt=failure == 'parser') + reader.timeout = 0 + scopes, recoveries = [], [] + class Runtime: + def native_scope(self, evidence): + value = reader.desktop_factory('shared') + value.cleanup_record = {'workers_closed': True} + original = value.start_process + first = not scopes + scopes.append(evidence) + def start(command, **options): + if first and failure == 'launch': + raise OSError('temporary process launch failed') + process = original(command, **options) + if first and failure == 'timeout': + process.poll = lambda: None + elif first and failure in {'crash', 'input_error'}: + process.poll = lambda: 0xC0000005 if failure == 'crash' else 2 + process.close = lambda: None + return process + value.start_process = start + return value + def recover_native_failure(self, path, evidence, reason): + proof = json.loads((evidence / 'native-retry-proof.json').read_text()) + assert path == project and reason + assert proof['native_read_only'] and proof['workers_closed'] + assert proof['project_path'] == str(project.resolve()) + assert set(proof['source_hashes']) == {str(project), str(project.parent / 'database/icdb.dat')} + assert released == [True] + recoveries.append(evidence) + return True + result = reader.read(project, ('OLD',), runtime=Runtime()) + if failure in {'launch', 'timeout', 'crash'}: + assert [row.refdes for row in result.components] == ['R1'] + assert len(scopes) == 2 and len(recoveries) == 1 + else: + assert result is None and len(scopes) == 1 and not recoveries + + @pytest.mark.parametrize('failure', ['malformed', 'timeout']) def test_safe_native_failure_allows_com_fallback_only_after_desktop_release(tmp_path, failure): project, reader, stop, calls, released = reader_fixture( diff --git a/tests/v1_1/test_service_plan_safety.py b/tests/v1_1/test_service_plan_safety.py index 375d0dd..f5fa7eb 100644 --- a/tests/v1_1/test_service_plan_safety.py +++ b/tests/v1_1/test_service_plan_safety.py @@ -1,4 +1,5 @@ from dataclasses import replace +import json from threading import Event import pytest @@ -73,3 +74,32 @@ def test_cancelled_service_inventory_does_not_save_any_targets(group): service.collect_target_inventory(task_id, ('OLD',), stop_event=stop) assert error.value.code == 'operation_cancelled' assert not store.list_target_plan(task_id) and not store.get_task(task_id).target_plan_mode + + +@pytest.mark.parametrize('entry', ['collect', 'prepare', 'execute', 'retry', 'recover', 'preview', 'inspect']) +def test_pending_resources_block_every_standalone_cad_service_before_target_or_worker_changes(group, tmp_path, entry): + service, store, tasks, targets = group + task_id = tasks[0].task_id + calls = [] + def collect(*args, **kwargs): + calls.append('read') + return (targets[task_id],) + service.for_project_task(task_id)._quick_run_preparer.collect_inventory = collect + record = service._logging.root / 'batch-resources/previous/session.json' + record.parent.mkdir(parents=True) + record.write_text(json.dumps({'version': 1, 'session_id': 'previous', 'state': 'blocked'})) + path = write_bom_template(tmp_path / 'scope.xlsx', ({'project_path': str(targets[task_id].project_path), + 'old_part_number': 'OLD', 'new_part_number': 'NEW', 'refdes': 'R1'},)) + invoke = { + 'collect': lambda: service.collect_target_inventory(task_id, ('OLD',)), + 'prepare': lambda: service.prepare_quick_run(task_id), + 'execute': lambda: service.execute_quick_run(task_id), + 'retry': lambda: service.prepare_failed_quick_run(task_id), + 'recover': lambda: service.recover_quick_run(task_id, confirmed=True), + 'preview': lambda: service.preview_bom_batch(task_id, path), + 'inspect': lambda: service.inspect_bom_batch(task_id, path), + }[entry] + with pytest.raises(TaskCenterError) as caught: + invoke() + assert caught.value.code == 'batch_resources_pending' + assert not calls and not store.list_target_plan(task_id)