fix: guard standalone CAD entries and preserve cleanup diagnostics
This commit is contained in:
1 parent
00861a59d6
commit
a94276494a
13 files changed
+269
-37
No files matched your search
@@ -55,11 +55,11 @@ def _assert_editable(service, task_id):
|
||||
raise TaskCenterError('target_plan_locked', f'工程任务当前不能修改 BOM 计划:{context.task.name}')
|
||||
|
||||
|
||||
def preview_batch(service, task_id, path, *, stop_event=None, on_progress=None):
|
||||
def preview_batch(service, task_id, path, *, stop_event=None, on_progress=None, runtime=None):
|
||||
"""Strict adapter to the same validation engine used by the import report."""
|
||||
from .bom_import_report import inspect_batch
|
||||
|
||||
report = inspect_batch(service, task_id, path, stop_event=stop_event, on_progress=on_progress)
|
||||
report = inspect_batch(service, task_id, path, stop_event=stop_event, on_progress=on_progress, runtime=runtime)
|
||||
if report.preview is not None:
|
||||
return report.preview
|
||||
failed = next((row for row in report.rows if row.status == 'failed'), None)
|
||||
|
||||
@@ -134,8 +134,12 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None,
|
||||
report = None
|
||||
try:
|
||||
with BatchCadRuntime(service._logging.root, 'bom_import', stop_event) as owned_runtime:
|
||||
report = inspect_batch(service, task_id, path, stop_event=stop_event,
|
||||
on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime)
|
||||
try:
|
||||
report = inspect_batch(service, task_id, path, stop_event=stop_event,
|
||||
on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime)
|
||||
except BomInspectionError as error:
|
||||
report = error.bom_report
|
||||
raise
|
||||
owned_runtime.result = report
|
||||
return report
|
||||
except BatchResourceCleanupError as error:
|
||||
|
||||
@@ -90,9 +90,11 @@ class BomImportMachine:
|
||||
return False
|
||||
cancelled_error = isinstance(error, TaskCenterError) and error.code == 'operation_cancelled'
|
||||
if error is not None and not cancelled_error:
|
||||
from .bom_import_report import BomInspectionError
|
||||
from .bom_import_report import BomImportReport
|
||||
reason = error.message if isinstance(error, TaskCenterError) else str(error)
|
||||
report = error.bom_report if isinstance(error, BomInspectionError) else None
|
||||
report = getattr(error, 'bom_report', None)
|
||||
if not isinstance(report, BomImportReport):
|
||||
report = None
|
||||
self.state = replace(self.state, phase=BomImportPhase.FAILED, report=report,
|
||||
message=f'本次 BOM 导入未完成:{reason[:240]};原有预览和编辑保留。')
|
||||
elif cancelled_error or self.state.phase is BomImportPhase.CANCELLING:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
|
||||
from .errors import TaskCenterError
|
||||
@@ -58,11 +59,15 @@ def read_component_inventory(path: Path, project_path: Path,
|
||||
|
||||
|
||||
def collect_target_inventory(preparer, project_path: Path,
|
||||
old_part_numbers: tuple[str, ...], *, stop_event=None, on_progress=None) -> tuple[ComponentTarget, ...]:
|
||||
old_part_numbers: tuple[str, ...], *, stop_event=None, on_progress=None, runtime=None) -> tuple[ComponentTarget, ...]:
|
||||
if not old_part_numbers:
|
||||
raise TaskCenterError("target_inventory", "at least one old Part Number is required")
|
||||
from .operation_control import check_cancelled, control_options
|
||||
check_cancelled(stop_event)
|
||||
rows = preparer.collect_inventory(project_path, old_part_numbers, **control_options(stop_event, on_progress))
|
||||
runtime_options = ({'runtime': runtime} if runtime is not None and any(
|
||||
parameter.name == 'runtime' or parameter.kind is inspect.Parameter.VAR_KEYWORD
|
||||
for parameter in inspect.signature(preparer.collect_inventory).parameters.values()) else {})
|
||||
rows = preparer.collect_inventory(project_path, old_part_numbers, **runtime_options,
|
||||
**control_options(stop_event, on_progress))
|
||||
check_cancelled(stop_event)
|
||||
return rows
|
||||
@@ -17,6 +17,15 @@ from .native_inventory_protocol import NativeInventoryUnsupported, read_ascii_di
|
||||
from .operation_control import check_cancelled, report_progress
|
||||
|
||||
|
||||
_RETRY_NATIVE_READ = object()
|
||||
_NATIVE_CRASH_CODES = frozenset({0xC0000005, 0xC000001D, 0xC0000094,
|
||||
0xC00000FD, 0xC0000374, 0xC0000409})
|
||||
|
||||
|
||||
class _NativeInventoryProcessError(NativeInventoryUnsupported):
|
||||
"""A launch, watchdog or explicit crash failure, separate from invalid input."""
|
||||
|
||||
|
||||
def _hash(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
@@ -80,10 +89,15 @@ class NativeInventoryReader:
|
||||
except (NativeInventoryUnsupported, OSError, ImportError, UnicodeError, TaskCenterError) as error:
|
||||
report_progress(on_progress, f'使用 CAD 核验位号:{str(error)[:160]}')
|
||||
return None
|
||||
return self._read(project, text, block, snapshot, database, tools, old_part_numbers,
|
||||
stop_event=stop_event, on_progress=on_progress, runtime=runtime)
|
||||
for attempt in range(2):
|
||||
result = self._read(project, text, block, snapshot, database, tools, old_part_numbers,
|
||||
stop_event=stop_event, on_progress=on_progress, runtime=runtime,
|
||||
allow_retry=runtime is not None and attempt == 0)
|
||||
if result is not _RETRY_NATIVE_READ:
|
||||
return result
|
||||
return None
|
||||
|
||||
def _read(self, project, text, block, snapshot, database, tools, old_part_numbers, *, stop_event, on_progress, runtime=None):
|
||||
def _read(self, project, text, block, snapshot, database, tools, old_part_numbers, *, stop_event, on_progress, runtime=None, allow_retry=False):
|
||||
token = uuid4().hex
|
||||
evidence = self.data_root / 'quick-run-history' / ('native-inventory-' + token)
|
||||
evidence.mkdir(parents=True, exist_ok=False)
|
||||
@@ -152,9 +166,31 @@ class NativeInventoryReader:
|
||||
if record['state'] == 'uncertain':
|
||||
raise TaskCenterError('inventory_cleanup', f'原生采集清理状态不确定:{evidence}') from error
|
||||
_verify_sources(source_hashes)
|
||||
record.update(state='fallback', reason=str(error))
|
||||
report_progress(on_progress, f'原生清单未能完整核对,改用 CAD 采集:{str(error)[:160]}')
|
||||
retry = False
|
||||
cleanup = getattr(desktop, 'cleanup_record', None)
|
||||
if (allow_retry and isinstance(error, _NativeInventoryProcessError)
|
||||
and clean and isinstance(cleanup, dict) and cleanup.get('workers_closed') is True):
|
||||
check_cancelled(stop_event)
|
||||
from .batch_resources import write_record
|
||||
write_record(evidence / 'native-retry-proof.json', {
|
||||
'version': 1, 'native_read_only': True, 'project_path': str(project),
|
||||
'source_hashes': {str(path): digest for path, digest in source_hashes.items()},
|
||||
'workers_closed': True, 'evidence_root': str(evidence.resolve()),
|
||||
})
|
||||
_verify_sources(source_hashes)
|
||||
retry = runtime.recover_native_failure(project, evidence, str(error))
|
||||
record.update(state='retrying' if retry else 'fallback', reason=str(error))
|
||||
if retry:
|
||||
result = _RETRY_NATIVE_READ
|
||||
report_progress(on_progress, '原生读取会话已安全释放,正在重新核验一次…')
|
||||
else:
|
||||
report_progress(on_progress, f'原生清单未能完整核对,改用 CAD 采集:{str(error)[:160]}')
|
||||
except TaskCenterError as error:
|
||||
if record['state'] == 'uncertain':
|
||||
try:
|
||||
(evidence / 'uncertain.flag').write_text(error.message, encoding='utf-8')
|
||||
except OSError as marker_error:
|
||||
record['uncertain_marker_error'] = str(marker_error)
|
||||
if record['state'] != 'uncertain':
|
||||
record.update(state='cancelled' if error.code == 'operation_cancelled' else 'failed', reason=error.message)
|
||||
raise
|
||||
@@ -182,19 +218,32 @@ class NativeInventoryReader:
|
||||
started = time.monotonic()
|
||||
metric = {'name': name, 'report': report_name, 'exit_code': None}
|
||||
record['tools'].append(metric)
|
||||
process = None
|
||||
try:
|
||||
with (output / (log_name + '.stdout.txt')).open('wb') as stdout, (
|
||||
output / (log_name + '.stderr.txt')).open('wb') as stderr:
|
||||
process = desktop.start_process(command, cwd=workspace, env=environment,
|
||||
stdout=stdout, stderr=stderr)
|
||||
try:
|
||||
process = desktop.start_process(command, cwd=workspace, env=environment,
|
||||
stdout=stdout, stderr=stderr)
|
||||
except OSError as error:
|
||||
raise _NativeInventoryProcessError(f'{name} 启动失败:{error}') from error
|
||||
deadline = started + self.timeout
|
||||
while process.poll() is None:
|
||||
check_cancelled(stop_event)
|
||||
if time.monotonic() >= deadline:
|
||||
raise NativeInventoryUnsupported(f'{name} 原生读取超时')
|
||||
raise _NativeInventoryProcessError(f'{name} 原生读取超时')
|
||||
time.sleep(0.05)
|
||||
metric['exit_code'] = process.poll()
|
||||
if metric['exit_code'] & 0xFFFFFFFF in _NATIVE_CRASH_CODES:
|
||||
raise _NativeInventoryProcessError(f'{name} 原生读取进程崩溃:{metric["exit_code"]}')
|
||||
if metric['exit_code'] != 0:
|
||||
raise NativeInventoryUnsupported(f'{name} 返回退出码 {metric["exit_code"]}')
|
||||
finally:
|
||||
metric['seconds'] = time.monotonic() - started
|
||||
if process is not None and callable(getattr(process, 'close', None)):
|
||||
try:
|
||||
process.close()
|
||||
except Exception as error:
|
||||
record.update(state='uncertain', cleanup_error=str(error))
|
||||
raise TaskCenterError('inventory_cleanup',
|
||||
f'原生读取进程及其子进程未能确认退出:{error}') from error
|
||||
@@ -338,7 +338,8 @@ class QuickRunPreparer:
|
||||
raise TaskCenterError("quick_run_uncertain", "a previous quick-run worker timed out or returned an uncorrelated status; reconcile the project before retrying")
|
||||
if self._process_runner is _run_process and not batch_mode:
|
||||
from .single_step import prepare_private_single_step
|
||||
prepare_private_single_step(self, task_id, projects, rules, copy_part_number_to_value)
|
||||
prepare_private_single_step(self, task_id, projects, rules, copy_part_number_to_value,
|
||||
**({'runtime': runtime} if runtime is not None else {}))
|
||||
batch_mode = True
|
||||
current_root = self._data_root / "quick-run"
|
||||
if batch_session is None and self.evidence_path(task_id) == current_root and (current_root / "status.tsv").is_file():
|
||||
|
||||
@@ -105,7 +105,7 @@ def require_current_cell_binding(snapshot, binding, project, metadata):
|
||||
return current
|
||||
|
||||
|
||||
def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=False, confirmed=False):
|
||||
def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=False, confirmed=False, runtime=None):
|
||||
if not confirmed:
|
||||
raise TaskCenterError('quick_run_recovery_confirmation', '核验恢复需要明确确认,不能作为自动重试执行。')
|
||||
maintain_quick_run_storage(preparer._data_root)
|
||||
@@ -227,7 +227,11 @@ def recover(preparer, task_id, projects, rules, *, copy_part_number_to_value=Fal
|
||||
launch = (str(preparer._cscript_resolver()), '//nologo', script_argument, root_argument,
|
||||
*_native_helper_launch_arguments(), job_id)
|
||||
try:
|
||||
result = preparer._process_runner(launch, IDLE_PROGRESS_TIMEOUT_SECONDS)
|
||||
from .quick_run import _run_process
|
||||
if runtime is not None and preparer._process_runner is _run_process:
|
||||
result = _run_process(launch, IDLE_PROGRESS_TIMEOUT_SECONDS, runtime=runtime)
|
||||
else:
|
||||
result = preparer._process_runner(launch, IDLE_PROGRESS_TIMEOUT_SECONDS)
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
raise TaskCenterError('quick_run_recovery', '恢复执行未能确认结束,原保护锁仍在;不要重新替换或删除证据。') from error
|
||||
(audit / 'worker-output.txt').write_text(result.stdout + '\n' + result.stderr, encoding='utf-8')
|
||||
|
||||
@@ -142,10 +142,11 @@ class TaskCenterService:
|
||||
return self._operation("export_bom_template", task_id,
|
||||
lambda: export_template(self._store, task_id, path))
|
||||
|
||||
def preview_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None):
|
||||
def preview_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, runtime=None):
|
||||
from .bom_batch import preview_batch
|
||||
return self._idle_quick_run_operation('preview_bom_batch', task_id,
|
||||
lambda: preview_batch(self, task_id, path, **control_options(stop_event, on_progress)))
|
||||
lambda: preview_batch(self, task_id, path, **control_options(stop_event, on_progress),
|
||||
**({'runtime': runtime} if runtime is not None else {})))
|
||||
|
||||
def save_bom_batch(self, preview, rows_by_task):
|
||||
from .bom_batch import save_batch
|
||||
@@ -388,10 +389,15 @@ class TaskCenterService:
|
||||
return self._operation("list_projects", task_id, lambda: self._store.list_projects(task_id))
|
||||
|
||||
def collect_target_inventory(self, task_id: str, old_part_numbers: tuple[str, ...] | None = None,
|
||||
*, stop_event=None, on_progress=None):
|
||||
*, stop_event=None, on_progress=None, runtime=None):
|
||||
if runtime is None:
|
||||
return self._owned_cad_operation('collect_inventory', lambda owned:
|
||||
self.collect_target_inventory(task_id, old_part_numbers,
|
||||
stop_event=stop_event, on_progress=on_progress, runtime=owned), stop_event)
|
||||
delegated = self._project_task_service(task_id)
|
||||
if delegated is not self:
|
||||
return delegated.collect_target_inventory(task_id, old_part_numbers, **control_options(stop_event, on_progress))
|
||||
return delegated.collect_target_inventory(task_id, old_part_numbers, runtime=runtime,
|
||||
**control_options(stop_event, on_progress))
|
||||
def collect():
|
||||
from .bom_batch import _assert_editable
|
||||
_assert_editable(self, task_id)
|
||||
@@ -409,7 +415,7 @@ class TaskCenterService:
|
||||
project_path=str(project[0].path)) as metric:
|
||||
report_progress(on_progress, f'正在采集工程:{project[0].display_name}')
|
||||
rows = collect_target_inventory(self._quick_run_preparer, project[0].path, parts,
|
||||
**control_options(stop_event, on_progress))
|
||||
runtime=runtime, **control_options(stop_event, on_progress))
|
||||
check_cancelled(stop_event)
|
||||
metric['completed_units'] = len(rows)
|
||||
with self._store.transaction():
|
||||
@@ -555,11 +561,14 @@ class TaskCenterService:
|
||||
|
||||
return self._operation("expected_probe_path", task_id, expected_path)
|
||||
|
||||
def prepare_quick_run(self, task_id: str) -> QuickRunResult:
|
||||
def prepare_quick_run(self, task_id: str, *, runtime=None) -> QuickRunResult:
|
||||
"""Create the legacy VBS handoff without starting a task-center trial."""
|
||||
if runtime is None:
|
||||
return self._owned_cad_operation('prepare_quick_run', lambda owned:
|
||||
self.prepare_quick_run(task_id, runtime=owned))
|
||||
delegated = self._project_task_service(task_id)
|
||||
if delegated is not self:
|
||||
return delegated.prepare_quick_run(task_id)
|
||||
return delegated.prepare_quick_run(task_id, runtime=runtime)
|
||||
def prepare() -> QuickRunResult:
|
||||
task = self._store.get_task(task_id)
|
||||
preparer = (self._quick_run_preparer.ensure_batch_handoff
|
||||
@@ -569,6 +578,7 @@ class TaskCenterService:
|
||||
self._store.list_projects(task_id),
|
||||
self._execution_rules(task_id),
|
||||
copy_part_number_to_value=task.copy_part_number_to_value,
|
||||
**({'runtime': runtime} if task.target_plan_mode else {}),
|
||||
)
|
||||
append_task_event(
|
||||
self._logging,
|
||||
@@ -624,11 +634,14 @@ class TaskCenterService:
|
||||
with _quick_run_lease(self._logging.root):
|
||||
return self._operation("set_value_policy", task_id, update)
|
||||
|
||||
def execute_quick_run(self, task_id: str, *, expected_item: tuple[str, str, str, str] | None = None) -> QuickRunStepResult:
|
||||
def execute_quick_run(self, task_id: str, *, expected_item: tuple[str, str, str, str] | None = None, runtime=None) -> QuickRunStepResult:
|
||||
"""Execute one replacement rule through restoration and verification."""
|
||||
if runtime is None:
|
||||
return self._owned_cad_operation('quick_run_step', lambda owned:
|
||||
self.execute_quick_run(task_id, expected_item=expected_item, runtime=owned))
|
||||
delegated = self._project_task_service(task_id)
|
||||
if delegated is not self:
|
||||
return delegated.execute_quick_run(task_id, expected_item=expected_item)
|
||||
return delegated.execute_quick_run(task_id, expected_item=expected_item, runtime=runtime)
|
||||
def execute() -> QuickRunStepResult:
|
||||
task = self._store.get_task(task_id)
|
||||
result = self._quick_run_preparer.execute_step(
|
||||
@@ -637,6 +650,7 @@ class TaskCenterService:
|
||||
self._execution_rules(task_id),
|
||||
copy_part_number_to_value=task.copy_part_number_to_value,
|
||||
**({"expected_item": expected_item} if expected_item is not None else {}),
|
||||
runtime=runtime,
|
||||
)
|
||||
self._synchronize_quick_run_progress(task_id)
|
||||
append_task_event(
|
||||
@@ -718,11 +732,14 @@ class TaskCenterService:
|
||||
finally:
|
||||
self._batch_feedback = None
|
||||
|
||||
def prepare_failed_quick_run(self, task_id: str) -> QuickRunProgress:
|
||||
def prepare_failed_quick_run(self, task_id: str, *, runtime=None) -> QuickRunProgress:
|
||||
"""Reset only failed projects and discard the finished queue handoff."""
|
||||
if runtime is None:
|
||||
return self._owned_cad_operation('prepare_failed_quick_run', lambda owned:
|
||||
self.prepare_failed_quick_run(task_id, runtime=owned))
|
||||
delegated = self._project_task_service(task_id)
|
||||
if delegated is not self:
|
||||
return delegated.prepare_failed_quick_run(task_id)
|
||||
return delegated.prepare_failed_quick_run(task_id, runtime=runtime)
|
||||
def prepare_retry() -> QuickRunProgress:
|
||||
task = self._store.get_task(task_id)
|
||||
if self._quick_run_preparer.has_interrupted(task_id):
|
||||
@@ -768,6 +785,7 @@ class TaskCenterService:
|
||||
self._store.list_projects(task_id),
|
||||
self._execution_rules(task_id),
|
||||
copy_part_number_to_value=task.copy_part_number_to_value,
|
||||
runtime=runtime,
|
||||
)
|
||||
return self._quick_run_preparer.progress(
|
||||
task_id,
|
||||
@@ -787,6 +805,13 @@ class TaskCenterService:
|
||||
return self._idle_quick_run_operation('start_new_round', task_id,
|
||||
lambda: start_new_round(self, task_id))
|
||||
|
||||
def _owned_cad_operation(self, kind, action, stop_event=None):
|
||||
from .batch_runtime import BatchCadRuntime
|
||||
with BatchCadRuntime(self._logging.root, kind, stop_event) as runtime:
|
||||
result = action(runtime)
|
||||
runtime.result = result
|
||||
return result
|
||||
|
||||
def pending_batch_resources(self):
|
||||
from .batch_runtime import pending_resources
|
||||
return pending_resources(self._logging.root)
|
||||
@@ -800,10 +825,10 @@ class TaskCenterService:
|
||||
return self._operation('cleanup_batch_resources', None,
|
||||
lambda: cleanup_pending_resources(self._logging.root))
|
||||
|
||||
def inspect_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, on_phase=None):
|
||||
def inspect_bom_batch(self, task_id: str, path: Path, *, stop_event=None, on_progress=None, on_phase=None, runtime=None):
|
||||
from .bom_import_report import inspect_batch
|
||||
return self._idle_quick_run_operation('inspect_bom_batch', task_id,
|
||||
lambda: inspect_batch(self, task_id, path, on_phase=on_phase,
|
||||
lambda: inspect_batch(self, task_id, path, on_phase=on_phase, runtime=runtime,
|
||||
**control_options(stop_event, on_progress)))
|
||||
|
||||
def _pause_quick_run_queue(self, task_id: str, message: str) -> None:
|
||||
@@ -860,10 +885,13 @@ class TaskCenterService:
|
||||
return delegated.has_interrupted_quick_run(task_id)
|
||||
return self._quick_run_preparer.has_interrupted(task_id)
|
||||
|
||||
def recover_quick_run(self, task_id: str, *, confirmed: bool = False) -> QuickRunStepResult:
|
||||
def recover_quick_run(self, task_id: str, *, confirmed: bool = False, runtime=None) -> QuickRunStepResult:
|
||||
if runtime is None:
|
||||
return self._owned_cad_operation('recover_quick_run', lambda owned:
|
||||
self.recover_quick_run(task_id, confirmed=confirmed, runtime=owned))
|
||||
delegated = self._project_task_service(task_id)
|
||||
if delegated is not self:
|
||||
return delegated.recover_quick_run(task_id, confirmed=confirmed)
|
||||
return delegated.recover_quick_run(task_id, confirmed=confirmed, runtime=runtime)
|
||||
from .recovery import recover
|
||||
|
||||
def reconcile() -> QuickRunStepResult:
|
||||
@@ -877,7 +905,7 @@ class TaskCenterService:
|
||||
raise TaskCenterError("quick_run_recovery", "工程包含手动执行阶段或失败记录,不能混用自动队列恢复。")
|
||||
result = recover(self._quick_run_preparer, task_id, projects,
|
||||
self._execution_rules(task_id), copy_part_number_to_value=task.copy_part_number_to_value,
|
||||
confirmed=confirmed)
|
||||
confirmed=confirmed, runtime=runtime)
|
||||
if task.status in {TaskStatus.PAUSED, TaskStatus.FAILED}:
|
||||
self._store.set_task_status(task_id, TaskStatus.READY, summary="interrupted quick-run reconciled")
|
||||
self._synchronize_quick_run_progress(task_id)
|
||||
|
||||
@@ -5,7 +5,7 @@ from .errors import TaskCenterError
|
||||
from .schematic_backup import create_schematic_backup
|
||||
|
||||
|
||||
def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_number_to_value):
|
||||
def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_number_to_value, *, runtime=None):
|
||||
progress = preparer.progress(task_id, projects, rules,
|
||||
copy_part_number_to_value=copy_part_number_to_value)
|
||||
if progress.queue_complete or progress.next_project_path is None:
|
||||
@@ -15,4 +15,5 @@ def prepare_private_single_step(preparer, task_id, projects, rules, copy_part_nu
|
||||
create_schematic_backup(preparer.data_root, progress.next_project_path,
|
||||
preparer.data_root / 'quick-run-history' / ('batch-' + uuid4().hex))
|
||||
preparer.ensure_batch_handoff(task_id, projects, rules,
|
||||
copy_part_number_to_value=copy_part_number_to_value)
|
||||
copy_part_number_to_value=copy_part_number_to_value,
|
||||
**({'runtime': runtime} if runtime is not None else {}))
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
from dx_productivity.trial_task_center import task_queue
|
||||
from dx_productivity.trial_task_center.errors import TaskCenterError
|
||||
from test_task_queue import Service
|
||||
from test_bom_batch import batch, records_for
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -88,3 +89,25 @@ def test_reader_requires_current_batch_project_checkpoint(tmp_path, runtime_modu
|
||||
_require_released_reader(tmp_path)
|
||||
assert caught.value.code == 'inventory_cleanup'
|
||||
assert not isinstance(_read_failure(tmp_path, 'read failed'), ReleasedProjectReadError)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('read_error', [False, True])
|
||||
def test_bom_cleanup_failure_preserves_completed_or_partial_report(batch, tmp_path, runtime_module, read_error):
|
||||
from dx_productivity.trial_task_center.bom_import_report import inspect_batch
|
||||
from dx_productivity.trial_task_center.bom_plan import write_bom_template
|
||||
Runtime, scopes = runtime_module
|
||||
Runtime.fail_cleanup = True
|
||||
service, store, tasks, targets, calls = batch
|
||||
if read_error:
|
||||
def fail(*args, **kwargs):
|
||||
raise TaskCenterError('inventory_cleanup', 'reader did not close')
|
||||
service.for_project_task(tasks[0].task_id)._quick_run_preparer.collect_inventory = fail
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
inspect_batch(service, tasks[0].task_id,
|
||||
write_bom_template(tmp_path / 'cleanup.xlsx', records_for(tasks, targets)))
|
||||
assert caught.value.code == 'batch_resources_pending'
|
||||
report = caught.value.bom_report
|
||||
assert report is not None and report.total_rows == 2
|
||||
assert report.failed_rows == (1 if read_error else 0)
|
||||
assert report.passed_rows == (0 if read_error else 2)
|
||||
assert len(scopes) == 1 and scopes[0].closed
|
||||
@@ -76,3 +76,16 @@ def test_failed_import_preserves_attached_row_report_and_retry_resets_it():
|
||||
assert machine.state.phase is Phase.FAILED and machine.state.report is report
|
||||
machine.start()
|
||||
assert machine.state.report is None and machine.state.phase is Phase.READING
|
||||
|
||||
|
||||
def test_resource_cleanup_failure_preserves_bom_row_report_after_cancellation():
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchResourceCleanupError
|
||||
machine = BomImportMachine()
|
||||
attempt = machine.start()
|
||||
machine.cancel(attempt)
|
||||
report = BomImportReport((), (), errors=('project read failed',))
|
||||
error = BatchResourceCleanupError('desktop cleanup failed')
|
||||
error.bom_report = report
|
||||
machine.finish(attempt, error=error)
|
||||
assert machine.state.phase is Phase.FAILED and machine.state.report is report
|
||||
assert 'desktop cleanup failed' in machine.state.message
|
||||
@@ -90,6 +90,78 @@ def test_native_reader_borrows_batch_desktop_and_closes_only_native_scope(tmp_pa
|
||||
assert len(evidence) == 1 and evidence[0].parent.name == 'quick-run-history'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fail_close', [False, True])
|
||||
def test_native_jobs_release_descendants_before_next_command_or_fallback(tmp_path, fail_close):
|
||||
project, reader, stop, calls, released = reader_fixture(tmp_path)
|
||||
factory = reader.desktop_factory
|
||||
events = []
|
||||
def desktop(*args, **options):
|
||||
value = factory(*args, **options)
|
||||
original = value.start_process
|
||||
def start(command, **kwargs):
|
||||
events.append('start')
|
||||
process = original(command, **kwargs)
|
||||
def close():
|
||||
events.append('close')
|
||||
if fail_close:
|
||||
raise OSError('native descendant remained alive')
|
||||
process.close = close
|
||||
return process
|
||||
value.start_process = start
|
||||
return value
|
||||
reader.desktop_factory = desktop
|
||||
if fail_close:
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
reader.read(project, ('OLD',))
|
||||
assert caught.value.code == 'inventory_cleanup'
|
||||
assert events == ['start', 'close']
|
||||
assert list((tmp_path / 'evidence').rglob('uncertain.flag'))
|
||||
else:
|
||||
assert reader.read(project, ('OLD',)).components
|
||||
assert events == ['start', 'close', 'start', 'close']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['launch', 'timeout', 'crash', 'input_error', 'parser'])
|
||||
def test_only_transient_native_process_failure_requests_one_proven_rebuild(tmp_path, failure):
|
||||
project, reader, stop, calls, released = reader_fixture(tmp_path, corrupt=failure == 'parser')
|
||||
reader.timeout = 0
|
||||
scopes, recoveries = [], []
|
||||
class Runtime:
|
||||
def native_scope(self, evidence):
|
||||
value = reader.desktop_factory('shared')
|
||||
value.cleanup_record = {'workers_closed': True}
|
||||
original = value.start_process
|
||||
first = not scopes
|
||||
scopes.append(evidence)
|
||||
def start(command, **options):
|
||||
if first and failure == 'launch':
|
||||
raise OSError('temporary process launch failed')
|
||||
process = original(command, **options)
|
||||
if first and failure == 'timeout':
|
||||
process.poll = lambda: None
|
||||
elif first and failure in {'crash', 'input_error'}:
|
||||
process.poll = lambda: 0xC0000005 if failure == 'crash' else 2
|
||||
process.close = lambda: None
|
||||
return process
|
||||
value.start_process = start
|
||||
return value
|
||||
def recover_native_failure(self, path, evidence, reason):
|
||||
proof = json.loads((evidence / 'native-retry-proof.json').read_text())
|
||||
assert path == project and reason
|
||||
assert proof['native_read_only'] and proof['workers_closed']
|
||||
assert proof['project_path'] == str(project.resolve())
|
||||
assert set(proof['source_hashes']) == {str(project), str(project.parent / 'database/icdb.dat')}
|
||||
assert released == [True]
|
||||
recoveries.append(evidence)
|
||||
return True
|
||||
result = reader.read(project, ('OLD',), runtime=Runtime())
|
||||
if failure in {'launch', 'timeout', 'crash'}:
|
||||
assert [row.refdes for row in result.components] == ['R1']
|
||||
assert len(scopes) == 2 and len(recoveries) == 1
|
||||
else:
|
||||
assert result is None and len(scopes) == 1 and not recoveries
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['malformed', 'timeout'])
|
||||
def test_safe_native_failure_allows_com_fallback_only_after_desktop_release(tmp_path, failure):
|
||||
project, reader, stop, calls, released = reader_fixture(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from dataclasses import replace
|
||||
import json
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
@@ -73,3 +74,32 @@ def test_cancelled_service_inventory_does_not_save_any_targets(group):
|
||||
service.collect_target_inventory(task_id, ('OLD',), stop_event=stop)
|
||||
assert error.value.code == 'operation_cancelled'
|
||||
assert not store.list_target_plan(task_id) and not store.get_task(task_id).target_plan_mode
|
||||
|
||||
|
||||
@pytest.mark.parametrize('entry', ['collect', 'prepare', 'execute', 'retry', 'recover', 'preview', 'inspect'])
|
||||
def test_pending_resources_block_every_standalone_cad_service_before_target_or_worker_changes(group, tmp_path, entry):
|
||||
service, store, tasks, targets = group
|
||||
task_id = tasks[0].task_id
|
||||
calls = []
|
||||
def collect(*args, **kwargs):
|
||||
calls.append('read')
|
||||
return (targets[task_id],)
|
||||
service.for_project_task(task_id)._quick_run_preparer.collect_inventory = collect
|
||||
record = service._logging.root / 'batch-resources/previous/session.json'
|
||||
record.parent.mkdir(parents=True)
|
||||
record.write_text(json.dumps({'version': 1, 'session_id': 'previous', 'state': 'blocked'}))
|
||||
path = write_bom_template(tmp_path / 'scope.xlsx', ({'project_path': str(targets[task_id].project_path),
|
||||
'old_part_number': 'OLD', 'new_part_number': 'NEW', 'refdes': 'R1'},))
|
||||
invoke = {
|
||||
'collect': lambda: service.collect_target_inventory(task_id, ('OLD',)),
|
||||
'prepare': lambda: service.prepare_quick_run(task_id),
|
||||
'execute': lambda: service.execute_quick_run(task_id),
|
||||
'retry': lambda: service.prepare_failed_quick_run(task_id),
|
||||
'recover': lambda: service.recover_quick_run(task_id, confirmed=True),
|
||||
'preview': lambda: service.preview_bom_batch(task_id, path),
|
||||
'inspect': lambda: service.inspect_bom_batch(task_id, path),
|
||||
}[entry]
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
invoke()
|
||||
assert caught.value.code == 'batch_resources_pending'
|
||||
assert not calls and not store.list_target_plan(task_id)
|
||||
Reference in new issue
Block a user