"""Serial Verify supervisor. Each CBB gets one isolated, bounded automation worker on a private Windows desktop. The worker reproduces the observed refresh flow: invoke the CBB's native Library Manager Verify action, bind the resulting ReusableCircuit PCB, verify the document identity, run Project Integration, save the Reusable Block cell and close. Optionally the official ``ReuseBlockImporter.exe -chkContents`` check runs first when a LibSpec and a DMS login configuration are supplied. Nothing here injects mouse or keyboard input or edits the central library. The native menu invocation happens on the worker's private desktop. """ from dataclasses import asdict, dataclass, field, replace from pathlib import Path import json import logging import os import platform import re import struct import subprocess import sys import time from uuid import uuid4 from .network_path import absolute_preserving_drive from .runtime_timing import RunTimings from .defaults import DEFAULT_TIMEOUT_SECONDS SUCCESS_TEXT = re.compile(r'content checking completed successfully', re.IGNORECASE) ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE) INVALID_NAME_CHARS = '\\/:*?"<>|' def _notify_progress(callback, *args): """A failed observer cannot change native execution or release facts.""" if callback is not None: try: callback(*args) except Exception: logging.getLogger(__name__).warning('Progress notification failed', exc_info=True) def _write_optional_evidence(path, value): try: path.write_text(json.dumps(value, ensure_ascii=False, default=str, indent=2), encoding='utf-8') except OSError: logging.getLogger(__name__).warning('Could not write diagnostic evidence %s', path, exc_info=True) @dataclass(frozen=True) class VerifyRequest: lmc_file: Path block_name: str evidence_dir: Path libspec: str = '' dms_login_config: str = '' mentor_bin: str = '' # PCB startup on EE7.9 includes iCDB recovery and COM registration. The # previous 30-second UI default was shorter than the observed startup # window on this machine and caused a false timeout before document attach. timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS logging_context: object | None = None # The production default stays on an isolated desktop. The explicit # interactive mode is reserved for a real-input compatibility test on # machines where EE7.9 cannot dispatch the tree menu on a private desktop. interactive_input: bool = False # The native Library Manager menu route is opt-in until a machine's menu # dispatch has been proven. The flag is serialized to the worker so a # Win10 compatibility run can select it explicitly. native_lm_verify: bool = False # A reserved iCDB range is a host-wide prerequisite failure. Do not # launch a PCB that is guaranteed to time out; batch orchestration will # record this item and continue with the next one. strict_port_check: bool = False # A mapped drive can be invisible to an elevated/private worker on Win10. # Preserve the parent session's mapping identity so the worker can restore # it before launching Library Manager/PCB. mapped_drive: str = '' mapped_remote_root: str = '' auto_repair: bool = False maintenance: bool = False post_repair_proof: object | None = field(default=None, repr=False, compare=False) diagnostic_only: bool = False auto_confirm_pcb_open: bool = False @dataclass(frozen=True) class VerifyResult: status: str stderr: str = '' stdout: str = '' evidence_path: Path | None = None safe_to_continue: bool = True library_flag: str = 'UNKNOWN' library_flag_detail: str = '' report_path: Path | None = None content_check_exit_code: int | None = None error_category: str = "" error_stage: str = "" bridge_attempts: tuple[dict, ...] = () timings_seconds: dict[str, float] = field(default_factory=dict) incomplete_timing_phases: tuple[str, ...] = () recovery_attempts: tuple[dict, ...] = () release_proof: dict | None = None write_outcome: dict | None = None session_record_path: Path | None = None def resolve_importer(mentor_bin: str | None = None) -> Path: """Configured bin first, then the EE7.9.5 environment defaults.""" candidates = [] if mentor_bin: configured = Path(mentor_bin) if configured.is_file() and configured.name.casefold() == 'reuseblockimporter.exe': candidates.append(configured.parent) else: candidates.append(configured) if configured.name.casefold() == 'sdd_home': candidates.append(configured / 'lm' / 'win32' / 'bin') from .mentor_paths import discover_sdd_home discovered = discover_sdd_home(mentor_bin) if discovered is not None: candidates.append(discovered / 'lm' / 'win32' / 'bin') for variable in ('SDD_LM_BIN', 'SDD_HOME'): value = os.environ.get(variable) if not value: continue root = Path(value) candidates.append(root if root.name.lower() == 'bin' else root/'lm'/'win32'/'bin') candidates.append(Path(r'D:\Mentor\7.9.5EE\SDD_HOME\lm\win32\bin')) for candidate in candidates: executable = candidate/'ReuseBlockImporter.exe' if executable.is_file(): return executable raise ValueError('ReuseBlockImporter.exe 未找到;请设置 SDD_LM_BIN') def build_content_check_command(request, executable) -> tuple[str, ...]: """The documented Logical-Physical contents check invocation.""" return (str(executable), '-lmcFile', str(request.lmc_file), '-blockName', request.block_name, '-logicalType', 'LP', '-chkContents', '-libSpec', str(request.libspec), '-dmsLoginConfig', str(request.dms_login_config)) def contents_report_path(lmc_file, block_name) -> Path: safe_name = re.sub(f'[{re.escape(INVALID_NAME_CHARS)}]', '_', str(block_name)) return Path(lmc_file).parent/'LogFiles'/f'restorerb_RULP_{safe_name}_contents.log' def classify_output(exit_code: int | None, output: str, *, timed_out: bool = False) -> str: """Never invent success: PASS needs the success text and no error marker.""" if timed_out: return 'TIMEOUT' from .icdb_recovery import parse_session_loss if parse_session_loss(output): return 'FAIL' diagnostic_text = re.sub(r'\b0\s+errors?\b', '', output, flags=re.I) if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(diagnostic_text): return 'PASS' if ERROR_TEXT.search(output): return 'FAIL' if exit_code not in (None, 0): return 'FAIL' return 'UNKNOWN' def preflight(request): """Validate the library entry and its ReusableCircuit sources before launch.""" library = absolute_preserving_drive(request.lmc_file) name = request.block_name if not name or name in ('.', '..') or any(character in name for character in INVALID_NAME_CHARS): raise ValueError('CBB 名称非法') if not library.is_file(): raise ValueError(f'中心库不存在: {library}') work = library.parent/'ReusableBlocksLP'/name/'Work' projects = list(work.glob('*.prj')) if len(projects) != 1: raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ') project = absolute_preserving_drive(projects[0]) from .backup_recovery import _project_text content = _project_text(project) refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M) if not refs: raise ValueError(f'{name}: 工程没有 PCBDesignPath') pcb = absolute_preserving_drive(project.parent/refs[0]) if not pcb.is_relative_to(absolute_preserving_drive(work)): raise ValueError(f'{name}: PCB 引用越出当前 CBB,停止操作: {pcb}') real_work = work.resolve() expected_work = library.parent.resolve() / 'ReusableBlocksLP' / name / 'Work' if os.path.normcase(str(real_work)) != os.path.normcase(str(expected_work)): raise ValueError(f'{name}: Work 路径通过链接指向其他工程,停止操作') for candidate in (project, pcb, pcb.parent/'pcb.prj', pcb.parent/'Layout', pcb.parent/'Layout'/'LayoutDB.lyt', work/'database', work/'CES'): if not candidate.resolve().is_relative_to(real_work): raise ValueError(f'{name}: 文件或目录链接越出当前 CBB,停止操作: {candidate}') if not pcb.is_file() or not (pcb.parent/'Layout'/'LayoutDB.lyt').is_file(): raise ValueError(f'{name}: 缺少 PCB 或布局数据库: {pcb}') return project, pcb def preflight_skip_reason(lmc_file, block_name): """Return a user-facing skip reason for an import-incomplete CBB. This is intentionally narrower than ``preflight``: only local source structure problems are excluded before a batch starts. Runtime failures such as ports, DMS, COM, or Project Integration remain normal per-item failures and are still attempted when the user queues them. """ library = absolute_preserving_drive(lmc_file) if not library.is_file(): return '' work = library.parent / 'ReusableBlocksLP' / str(block_name) / 'Work' if not work.is_dir(): return f'{block_name}: 缺少 Work 工程目录' request = VerifyRequest(library, str(block_name), library.parent / '.cbb-preflight-evidence') try: preflight(request) except Exception as error: detail = str(error) if any(marker in detail for marker in ('缺少或无法唯一确定 Work 下的工程 PRJ', '工程没有 PCBDesignPath', '缺少 PCB 或布局数据库')): return detail return '' def _run_process(command, timeout_seconds, cwd, *, evidence_root=None): from .content_check import run_content_check return run_content_check(command, timeout_seconds, cwd, evidence_root=evidence_root) def _stop_worker(process, *, timeout=5, wait_for_native_write=True): """Request stop without interrupting a possibly unreturned native call.""" try: if hasattr(process, 'request_stop'): process.request_stop() except Exception: pass try: if getattr(process, '_stop_wait_complete', False): return process.poll() is not None process.wait(timeout=timeout) except Exception: pending = getattr(process, 'native_write_pending', lambda: False)() wait_required = wait_for_native_write() if callable(wait_for_native_write) else wait_for_native_write if wait_required and pending is not False and not getattr(process, '_stop_wait_complete', False): from .defaults import NATIVE_WRITE_STOP_WAIT_SECONDS try: process.wait(timeout=max(0, NATIVE_WRITE_STOP_WAIT_SECONDS - timeout)) except Exception: pass process._stop_wait_complete = True try: return process.poll() is not None except Exception: return False def _normalize_bridge_result(probe): from .status_bridge import BridgeResult if not isinstance(probe, BridgeResult): detail = str(probe[1]) if isinstance(probe, tuple) and len(probe) > 1 else str(probe) return BridgeResult('MALFORMED', '', None, 'BRIDGE_RESULT_INVALID: 非权威桥接器结果;' + detail) if probe.status in {'READY', 'PENDING'} and ( probe.source != 'IsVerifiedReuseBlock' or probe.raw_status != (1 if probe.status == 'READY' else 0)): return BridgeResult('MALFORMED', probe.source, probe.raw_status, 'BRIDGE_RESULT_INVALID: status/source/raw_status 不一致') return probe def read_library_flag_with_retry(reader, library, block_name, *, attempts=3, delays=(0.0, 3.0, 5.0), sleep=None, on_attempt=None): """Bounded read-only bridge retries; never repeat Verify or promote PENDING.""" from .status_bridge import BridgeResult sleep = sleep or time.sleep last = BridgeResult('UNAVAILABLE', '', None, '状态桥没有返回结果') for index in range(max(1, int(attempts))): if index > 0: delay = delays[min(index, len(delays)-1)] if delays else 0 sleep(max(0.0, float(delay))) started = time.monotonic() try: last = _normalize_bridge_result(reader(library, block_name)) except Exception as error: last = BridgeResult('UNAVAILABLE', '', None, 'BRIDGE_UNAVAILABLE: ' + str(error)) if on_attempt: on_attempt({'attempt': index + 1, 'status': last.status, 'source': last.source, 'raw_status': last.raw_status, 'detail': last.detail, 'query_seconds': round(time.monotonic()-started, 6)}) if last.status in {'READY', 'MALFORMED'}: return last return last def classify_failure_category(detail): text = str(detail or '') lower = text.casefold() if 'pcb_version_choice_required' in lower: return 'PCB_VERSION_CHOICE_REQUIRED' from .icdb_recovery import ERROR_CODE, parse_session_loss if ERROR_CODE.casefold() in lower or parse_session_loss(text): return ERROR_CODE if 'back annotation is not allowed on verified (or once verified) reusable blocks.' in lower: # Preserve the native integration failure even when shutdown adds a # secondary release warning. This category does not grant permission # to reimport or change safe_to_continue. return 'CBB_REIMPORT_REQUIRED' if any(code in lower for code in ( 'pcb_release_unconfirmed', 'icdb_release_unconfirmed', 'icdb_session_unconfirmed', 'pcb_session_active', 'database_recovery_required')): return 'DATABASE_SAFETY_STOP' if 'tf_floatinglangbar_wndtitle' in lower or 'cicerouiwndframe' in lower: return 'TOOL_FALSE_DIALOG' if 'could not load job preferences db' in lower or 'unable to create local cell library' in lower: return 'MENTOR_LOCAL_CELL_LIBRARY' if ('database_load_failed' in lower or 'problem encountered while opening one of the database files' in lower or 'database load is being terminated' in lower): return 'MENTOR_DATABASE_LOAD_FAILED' if 'project integration error' in lower or 'project_integration_failed' in lower: return 'MENTOR_PROJECT_INTEGRATION' if 'save_reusable_block_failed' in lower or 'error saving the reusable block' in lower: return 'MENTOR_SAVE_FAILED' if 'design_server_failed' in lower or 'could not run server' in lower or 'could not bring it alive' in lower: return 'ENVIRONMENT_DESIGN_SERVER' if 'bridge_unavailable' in lower or '权威状态读取失败' in text: return 'BRIDGE_UNAVAILABLE' if 'bridge_result_invalid' in lower: return 'BRIDGE_RESULT_INVALID' if 'pending' in lower and 'isverifiedreuseblock' in lower: return 'MENTOR_VERIFY_NOT_COMMITTED' if 'cert' in lower and ('incomplete' in lower or '成果' in text): return 'TOOL_CERT_DIAGNOSTIC' if 'unknown_dialog' in lower or 'dialog' in lower: return 'TOOL_DIALOG_SUPERVISOR' if 'timeout' in lower or '超时' in text: return 'TOOL_TIMEOUT' return 'UNCLASSIFIED' def read_library_flag_if_available(library, block_name): """Read the authoritative Mentor Verify state through the x86 bridge. Unit tests replace this external boundary; production never accepts Cert files or legacy tree readers as a substitute for this gold standard. """ from .status_bridge import StatusBridgeClient return StatusBridgeClient( env={'CBB_STATUS_BRIDGE_ALLOW_PRIVATE': '1'} ).query(Path(library), block_name) def apply_post_verify_status(result, bridge_result): """Only the authoritative bridge can confirm READY, never a Cert/tuple/icon.""" if result.status != 'PASS': return result # a pre-existing READY cannot erase a real execution error bridge_result = _normalize_bridge_result(bridge_result) if bridge_result.status == 'READY': return replace(result, library_flag='GREEN', library_flag_detail=( f'READY ({bridge_result.source}, raw={bridge_result.raw_status})')) detail = bridge_result.detail or ( f'执行已完成,但权威状态仍为 {bridge_result.status} ' f'({bridge_result.source}, raw={bridge_result.raw_status})') pending = bridge_result.status == 'PENDING' return replace(result, status='FAIL', stderr='\n'.join(part for part in (result.stderr, detail) if part), library_flag='NOT_GREEN' if pending else 'UNKNOWN', library_flag_detail=detail, error_category=('MENTOR_VERIFY_NOT_COMMITTED' if pending else 'BRIDGE_RESULT_INVALID' if bridge_result.status == 'MALFORMED' else 'BRIDGE_UNAVAILABLE'), error_stage='library_flag', safe_to_continue=True) def worker_command(request_path): if getattr(sys, 'frozen', False): return [sys.executable, '--verify-worker', str(request_path)] return [sys.executable, '-m', 'cbb_refresh_verify', '--verify-worker', str(request_path)] def phase_for_event(event, current=None): from .dialog_models import DialogPhase stage = str(event.get('stage', '')) mapping = { 'lm_launch': DialogPhase.PCB_STARTING, 'lm_started': DialogPhase.PCB_STARTING, 'lm_ready': DialogPhase.PCB_STARTING, 'lm_session_ready': DialogPhase.PCB_STARTING, 'verify_invoke': DialogPhase.PCB_STARTING, 'verify_invoke_returned': DialogPhase.PCB_STARTING, 'pcb_launch': DialogPhase.PCB_STARTING, 'pcb_started': DialogPhase.PCB_STARTING, 'com_wait': DialogPhase.DOCUMENT_ATTACH, 'identity_wait': DialogPhase.DOCUMENT_ATTACH, 'identity_verified': DialogPhase.IDENTITY_CHECK, 'license_key_requested': DialogPhase.LICENSE_ACQUISITION, 'license_token_received': DialogPhase.LICENSE_ACQUISITION, 'document_validated': DialogPhase.VALIDATE, 'integration_status': DialogPhase.PROJECT_INTEGRATION, 'integration_action': DialogPhase.PROJECT_INTEGRATION, 'integration_error': DialogPhase.PROJECT_INTEGRATION, 'save_start': DialogPhase.SAVE, 'save_helper': DialogPhase.SAVE, 'save_progress': DialogPhase.SAVE, 'save_quit_recovered': DialogPhase.CLOSE, 'saved_and_closed': DialogPhase.CLOSE, 'pcb_exit_recovered': DialogPhase.CLOSE, 'complete': DialogPhase.CLOSE, } return mapping.get(stage, current or DialogPhase.PCB_STARTING) def _diagnostic_close_event(job, event, target_pid): close_id = event.get('close_id') return (job.get('diagnostic_only') is True and event.get('stage') == 'diagnostic_close_requested' and target_pid is not None and event.get('pcb_pid') == target_pid and type(event.get('hwnd')) is int and event['hwnd'] > 0 and isinstance(close_id, str) and len(close_id) == 32 and all(c in '0123456789abcdef' for c in close_id) and bool(job.get('operation_id')) and event.get('operation_id') == job['operation_id'] and all(job.get(key) and event.get(key) == job[key] for key in ('project', 'pcb', 'desktop'))) def _acknowledge_diagnostic_close(job, event, stop): from .operation_queue import check_active check_active() if stop is not None and stop.is_set() or Path(job['cancel_signal']).exists(): raise InterruptedError('VERIFY_CANCELLED: diagnostic handoff cancelled') ack = Path(job['root'])/'diagnostic-close-ack.json' pending = ack.with_suffix('.pending') pending.write_text(json.dumps({key: event[key] for key in ('pcb_pid', 'hwnd', 'operation_id', 'desktop', 'project', 'pcb', 'close_id')}), encoding='utf-8') check_active() if stop is not None and stop.is_set() or Path(job['cancel_signal']).exists(): raise InterruptedError('VERIFY_CANCELLED: diagnostic handoff cancelled') pending.replace(ack) def _close_intent_after_poll(job, events, offset, target_pid): if not job.get('diagnostic_only') or not events.exists(): return None with events.open(encoding='utf-8') as stream: stream.seek(offset) for line in stream: try: event = json.loads(line) except json.JSONDecodeError: break if _diagnostic_close_event(job, event, target_pid): return event return None def classify_verify_dispatch(*, verify_invoked, invoke_returned, target_pid, observation): """Classify a returned Verify command that never started its PCB. UI Automation can report a successful ``InvokePattern`` call even when Library Manager only closed the popup and did not dispatch its native MFC command. A returned menu call is therefore not a success signal. Keep this predicate small and evidence based so the supervisor can distinguish a dispatch failure from a generic inactivity timeout without changing the underlying automation route. """ if not (verify_invoked and invoke_returned) or target_pid is not None: return None return ( 'FAIL', 'VERIFY_DISPATCH_FAILED: LMC Verify 菜单调用已返回,但未启动目标 ReusableCircuit PCB', ) def _write_failure_windows(root, snapshots): if not snapshots: return records = [dict(time=time.strftime('%Y-%m-%d %H:%M:%S'), **snapshot.to_record()) for snapshot in snapshots] (root/'window_snapshots.jsonl').write_text( ''.join(json.dumps(record, ensure_ascii=False) + '\n' for record in records), encoding='utf-8') def _write_process_tree(root, process, target_pid): record = { 'time': time.strftime('%Y-%m-%d %H:%M:%S'), 'worker_pid': getattr(process, 'pid', None), 'companion_pids': list(getattr(process, 'companion_pids', ())), 'pcb_pid': target_pid, } temporary = root/'process_tree.json.tmp' temporary.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8') temporary.replace(root/'process_tree.json') def runtime_fingerprint(mentor_bin=''): """Return host/runtime facts needed to compare Win10 and Win11 runs.""" result = { 'platform': platform.platform(), 'system': platform.system(), 'release': platform.release(), 'version': platform.version(), 'machine': platform.machine(), 'python_version': platform.python_version(), 'python_bitness': struct.calcsize('P') * 8, 'mentor_bin': str(mentor_bin or ''), 'uia_backend': 'pywinauto.win32', } if os.name == 'nt': try: import winreg with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Microsoft\Windows NT\CurrentVersion') as key: for name, output_name in ( ('ProductName', 'windows_product_name'), ('CurrentBuild', 'windows_current_build'), ('UBR', 'windows_ubr'), ): try: result[output_name] = winreg.QueryValueEx(key, name)[0] except OSError: pass except (ImportError, OSError): pass result['windows_build_family'] = windows_build_family( result.get('windows_current_build')) from .mentor_paths import describe_installation, discover_sdd_home result['mentor_installation'] = describe_installation( discover_sdd_home(mentor_bin)) return result def windows_build_family(build): """Classify Windows from the build number, not ProductName. Windows 11 commonly keeps ``ProductName=Windows 10 Pro`` for compatibility. The first public Windows 11 build is 22000, so the build number is the stable discriminator for compatibility reports. """ try: number = int(str(build or '').split('.', 1)[0]) except (TypeError, ValueError): return 'unknown' if number >= 22000: return 'Windows 11' if number >= 10240: return 'Windows 10' return 'unknown' def native_lm_verify_enabled(): """Read the explicit compatibility switch for the native LM route.""" return os.environ.get('CBB_NATIVE_LM_VERIFY', '').strip().casefold() in { '1', 'true', 'yes', 'on', } def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, launcher_pid=None, progress=None): """Record native release or retention after automation has stopped.""" from .mentor_adapter import cleanup_verify_processes event_path = Path(root) / 'events.jsonl' def emit(**event): event = { 'time': time.strftime('%Y-%m-%d %H:%M:%S'), 'source': 'supervisor_cleanup', **event, } try: with event_path.open('a', encoding='utf-8') as stream: stream.write(json.dumps(event, ensure_ascii=False) + '\n') except OSError: pass if progress: _notify_progress(progress, event) return cleanup_verify_processes( lm_pid, target_pid, launcher_pid, expected_pcb=pcb, baseline_pids=pcb_baseline, emit=emit, ) def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None, admission_timeout_seconds=10): from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop if stop is not None and stop.is_set() or context is not None and context.stop.is_set(): return VerifyResult('CANCELLED', stderr='已取消,未启动原生操作。', safe_to_continue=True) options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt) bound = current_context() if bound is None: # Management may call Verify while holding a same-thread lease. Do not # transfer that transaction to a different thread. import threading from .library_lock import _guard, _owners with _guard: nested = any(v['thread'] == threading.get_ident() and v['depth'] > 0 for v in _owners.values()) if not nested: from .operation_queue import request_library_keys bound = context or BatchTaskContext(library=str(request.lmc_file), aliases=tuple(sorted(request_library_keys(request))), targets=(request.block_name,)) try: return bound.run(lambda: run_verify(request, stop=stop, **options), stop=stop, timeout_seconds=admission_timeout_seconds) except TaskHandoff as error: result = _handoff_verify_result(error) _notify_progress(bound.guard_callback(progress) if progress else None, dict(stage='task_handoff', operation_id=bound.operation_id, library=bound.library, status=result.status, resource_scope=error.scope)) return result if bound is not None and progress is not None: options['progress'] = bound.guard_callback(progress) return _run_verify_task(request, stop=task_stop(stop), **options) def _handoff_verify_result(error): known = error.result if isinstance(error.result, VerifyResult) else None result = known or VerifyResult('CANCELLED' if error.cancelled else 'TIMEOUT') return replace(result, status='FAIL' if result.status == 'PASS' else result.status, stderr=result.stderr or str(error), safe_to_continue=False, error_stage=result.error_stage or 'admission_or_finalization', error_category=result.error_category or 'TASK_HANDOFF', release_proof=dict(lease_state='PENDING', resource_scope=error.scope, blocked_resources=error.scope.get('resources', []))) def _run_verify_task(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None): # Admission/finalization keep only the library lease. Native launch paths # acquire the host channel after slow metadata has returned. from .operation_queue import check_active from .com_runtime import initialize_com, uninitialize_com initialized = initialize_com() library_lock = None previous_release = None release_confirmed = True remaining_clients = () def clients_released(): if release_confirmed: return True from .library_lock import _supervisor_alive return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients) try: check_active() if request.maintenance: from .maintenance_entry import run_icdb_check return run_icdb_check(request, progress=progress, stop=stop) from .library_lock import LibraryLock, LibraryBusy, recheck_previous_release import pywintypes if Path(request.lmc_file).is_file(): try: previous_release = recheck_previous_release(request.lmc_file, Path(request.evidence_dir)/'previous-release', block_name=request.block_name, progress=progress, stop=stop) check_active() library_lock = LibraryLock(request.lmc_file, checkpoint=False, block_name=request.block_name, safe_to_release=clients_released) library_lock.__enter__() check_active() except LibraryBusy as error: return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE') except (OSError, ValueError, RuntimeError, pywintypes.error) as error: return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_ACCESS_FAILED') release_confirmed = False result = _run_verify_serialized( request, progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop) facts = result.write_outcome or {} if (not request.diagnostic_only and not request.auto_repair and request.post_repair_proof is None and result.status == 'FAIL' and result.safe_to_continue and result.evidence_path and result.library_flag == 'NOT_GREEN' and result.error_stage == 'library_flag' and result.error_category == 'MENTOR_VERIFY_NOT_COMMITTED' and all(facts.get(key) == value for key, value in ( ('save_state','COMMITTED'), ('close_state','CLOSED'), ('helper_state','EXITED'), ('resource_state','RELEASED'), ('interrupted',False))) and not (stop is not None and stop.is_set())): # A new reusable design can save its Work data on the first open, # then publish Cert on the next native Save. Never retry an open/write failure. followup_path = result.evidence_path.with_name('commit-followup.json') followup = dict(kind='NATIVE_COMMIT_FOLLOWUP', status='STARTED', library=str(request.lmc_file), block=request.block_name, max_additional_runs=1, original_evidence=str(result.evidence_path)) initial_seconds = result.timings_seconds.get('total', 0.0) release_confirmed = True # Completed first run stays releasable until the next native attempt. try: followup_path.write_text(json.dumps(followup, ensure_ascii=False, indent=2), encoding='utf-8') except OSError as error: _notify_progress(progress, {'stage':'commit_followup_unavailable', 'block':request.block_name, 'detail':str(error)}) else: check_active() _notify_progress(progress, {'stage':'commit_followup', 'block':request.block_name, 'detail':'原生已保存关闭并释放,但 READY 尚未提交;补充一次原生校验。'}) check_active() release_confirmed = False result = _run_verify_serialized(request, progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop) followup.update(status='READY' if result.status == 'PASS' and result.library_flag == 'GREEN' and result.safe_to_continue else 'UNCONFIRMED', final_evidence=str(result.evidence_path or '')) _write_optional_evidence(followup_path, followup) second_seconds = result.timings_seconds.get('total', 0.0) result = replace(result, recovery_attempts=(*result.recovery_attempts, followup), timings_seconds={**result.timings_seconds, 'initial_verify':initial_seconds, 'commit_followup':second_seconds, 'total':initial_seconds+second_seconds}) if result.evidence_path: _write_optional_evidence(result.evidence_path, {'block_name':request.block_name, **asdict(result)}) if (request.auto_repair and result.status not in {'TIMEOUT', 'BLOCKED'} and result.error_category == 'ICDB_REPAIR_REQUIRED' and result.evidence_path): from .native_repair import attempt_recovery try: diagnosis = json.loads((result.evidence_path.parent/'diagnostics/icdb-recovery.json').read_text('utf-8')) original = result snapshot = original.evidence_path.with_name('pre-repair-result.json') snapshot.write_text(json.dumps(asdict(original), ensure_ascii=False, default=str, indent=2), encoding='utf-8') recovery = attempt_recovery(request, diagnosis, result.evidence_path.parent, progress=progress, stop=stop) recovery = {**recovery, 'original_error': original.stderr, 'original_evidence': str(original.evidence_path), 'original_result_snapshot': str(snapshot), 'original_stage': original.error_stage, 'native_status': recovery.get('native_status', recovery.get('status')), 'validation_passed': False} if recovery.get('status') == 'REPAIRED': from .native_repair import native_success_confirmed, select_repair_target try: confirmed = native_success_confirmed(select_repair_target(request, diagnosis), recovery) except (ValueError, OSError): confirmed = False if not confirmed: release = recovery.get('release') recovery.update(status='UNCONFIRMED', release={**(release if isinstance(release, dict) else {}), 'safe_to_continue': False}, detail='NATIVE_REPAIR_EVIDENCE_UNCONFIRMED: 原生成功记录不完整或矛盾,未重试校验') if (recovery.get('status') == 'REPAIRED' and recovery.get('release', {}).get('safe_to_continue') is True): result = replace(original, recovery_attempts=(recovery,)) recovery['status'] = 'REPAIR_VALIDATION_FAILED' result = _run_verify_serialized( replace(request, auto_repair=False, maintenance=False), progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop) recovery['validation_evidence'] = str(result.evidence_path or '') recovery['validation_passed'] = (result.status == 'PASS' and result.library_flag == 'GREEN' and result.safe_to_continue) if not recovery['validation_passed']: recovery['status'] = 'REPAIR_VALIDATION_FAILED' if result.status == 'PASS': result = replace(result, status='FAIL', stderr=(result.stderr+'\nREPAIR_VALIDATION_UNCONFIRMED: 未取得校验 GREEN').strip()) else: recovery['status'] = 'REPAIRED' else: # A failed local database must not block unrelated CBBs once # every owner is released. Re-check target ownership here so # an incomplete repair record cannot authorize the queue. safe = False if (recovery.get('status') in {'MISSING_HEALTHY_BACKUP', 'NATIVE_REPAIR_FAILED'} and recovery.get('evidence', {}).get('failure_complete') is True and recovery.get('release', {}).get('safe_to_continue') is True): from .native_repair import select_repair_target try: target = select_repair_target(request, diagnosis) safe = (Path(recovery.get('project', '')).resolve() == target.project and Path(recovery.get('database', '')).resolve() == target.database) except (ValueError, OSError): pass result = replace(result, safe_to_continue=safe, stderr=(result.stderr + '\n' + str(recovery.get('status', 'REPAIR_UNCONFIRMED')) + ': ' + str(recovery.get('detail', '原生 Repair 未完成恢复'))).strip()) result = replace(result, recovery_attempts=(recovery,)) result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)}, ensure_ascii=False, default=str, indent=2), encoding='utf-8') except Exception as error: result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False, stderr=(result.stderr+f'\nREPAIR_UNCONFIRMED: {error}').strip()) if result.evidence_path: try: result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)}, ensure_ascii=False, default=str, indent=2), encoding='utf-8') except OSError: pass # Keep the native/repair errors even if evidence storage fails. release_confirmed = result.safe_to_continue if not release_confirmed and result.evidence_path: try: release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8')) release_confirmed = (release.get('worker_exited') is True and release.get('safe_to_continue') is True and release.get('block') == request.block_name and Path(release['library']).resolve() == Path(request.lmc_file).resolve()) if (release.get('worker_exited') is True and not release.get('error') and not release.get('close_errors') and not release.get('automation_release', {}).get('error') and release.get('block') == request.block_name and Path(release['library']).resolve() == Path(request.lmc_file).resolve()): remaining_clients = tuple(pid for pid in release.get('remaining_pids', []) if type(pid) is int and pid > 0) except (OSError, ValueError, KeyError): pass if previous_release is not None: result = replace(result, release_proof=previous_release) if result.evidence_path: _write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)}) return result finally: try: if library_lock is not None: library_lock.__exit__(None, None, None) finally: uninitialize_com(initialized) def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None): from .operation_queue import remember_result timing = RunTimings() result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop) if result.evidence_path: try: facts = json.loads(result.evidence_path.with_name('write-outcome.json').read_text('utf-8')) session = result.evidence_path.with_name('session.json') result = replace(result, write_outcome=facts, session_record_path=session if session.is_file() else None) except (OSError, ValueError): pass if not request.auto_repair and result.evidence_path: try: release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8')) if (release.get('block') == request.block_name and Path(release['library']).resolve() == Path(request.lmc_file).resolve()): released = release.get('worker_exited') is True and release.get('safe_to_continue') is True result = replace(result, safe_to_continue=released) if not released and result.status == 'PASS': result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。') except (OSError, ValueError, KeyError): pass remember_result(result) if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path: job_path = result.evidence_path.with_name('request.json') if job_path.is_file(): timing.begin('failure_diagnostics') try: from .verify_diagnostics import capture_failure_logs_with_timeout capture_failure_logs_with_timeout(job_path) except Exception: # Never replace the underlying Mentor/bridge/timeout failure. pass finally: timing.end('failure_diagnostics') if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}: from .icdb_recovery import ERROR_CODE, RECOVERY_ACTION, parse_recovery_request, recovery_detail recovery = parse_recovery_request(result.stderr) if recovery is None and result.evidence_path: try: saved = json.loads(result.evidence_path.with_name('diagnostics').joinpath( 'icdb-recovery.json').read_text('utf-8')) recovery = parse_recovery_request(saved.get('native_message', '')) except (OSError, ValueError, AttributeError): pass if recovery or ERROR_CODE.casefold() in result.stderr.casefold(): detail = result.stderr if recovery and RECOVERY_ACTION not in detail: detail = (detail + '\n' + recovery_detail(recovery)).strip() result = replace(result, stderr=detail, error_category=ERROR_CODE) if recovery and result.evidence_path: try: record_path = result.evidence_path.parent/'diagnostics/icdb-recovery.json' record_path.parent.mkdir(parents=True, exist_ok=True) if not record_path.is_file(): record_path.write_text(json.dumps(recovery, ensure_ascii=False, indent=2), encoding='utf-8') except OSError: pass durations, unfinished = timing.snapshot() result = replace(result, timings_seconds=durations, incomplete_timing_phases=unfinished, error_category=(result.error_category or classify_failure_category(result.stderr) if result.status not in {"PASS", "SKIPPED"} else ""), error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution') if result.status not in {"PASS", "SKIPPED"} else "")) remember_result(result) if result.evidence_path: _write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)}) _write_optional_evidence(result.evidence_path.with_name('performance.json'), { 'block_name': request.block_name, 'status': result.status, 'unit': 'seconds', 'timings_seconds': durations, 'incomplete_timing_phases': unfinished, 'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)', 'notes': 'phase totals overlap action/call subphases; skipped phases are absent', }) if progress: _notify_progress(progress, {'stage': 'performance', 'block': request.block_name, 'timings_seconds': durations, 'incomplete_timing_phases': unfinished, 'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())}) if request.logging_context: try: request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s', request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations) except Exception: logging.getLogger(__name__).warning('Operation logging failed', exc_info=True) return result def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None): """Refresh one CBB and record its evidence. ``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``. After successful execution, only IsVerifiedReuseBlock READY/raw=1 confirms PASS/GREEN; Cert or icon observations never substitute for it. ``port_check`` is injectable so tests do not depend on the host's port reservations. """ root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12]) try: root.mkdir(parents=True, exist_ok=False) except OSError as error: return VerifyResult('FAIL', stderr=f'无法创建操作记录目录:{error}') fingerprint = runtime_fingerprint(request.mentor_bin) _write_optional_evidence(root/'runtime.json', fingerprint) if progress: _notify_progress(progress, {'stage': 'runtime_fingerprint', 'block': request.block_name, **fingerprint}) result_path = root/'result.json' runner = process_runner or (lambda command, timeout, cwd: _run_process(command, timeout, cwd, evidence_root=root)) def finish(result): # Only reclaim auxiliary backups after confirmed success. Failed runs # retain every available original file for incident investigation. try: from .mentor_adapter import cleanup_evidence_backup if result.status == 'PASS' and result.safe_to_continue: cleanup_evidence_backup(root) except Exception: # Cleanup must never hide or change the verification result. pass from .operation_queue import remember_result return remember_result(replace(result, evidence_path=result_path)) if stop is not None and stop.is_set(): return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。')) timing.begin('preflight') try: project, pcb = preflight(request) except Exception as error: return finish(VerifyResult('BLOCKED', stderr=str(error))) finally: timing.end('preflight') content_check_enabled = not request.diagnostic_only and bool(str(request.libspec).strip() and str(request.dms_login_config).strip()) if content_check_enabled: try: executable = resolve_importer(request.mentor_bin) except Exception as error: return finish(VerifyResult('BLOCKED', stderr=str(error))) command = build_content_check_command(request, executable) if progress: _notify_progress(progress, {'stage': 'content_check_start', 'block': request.block_name, 'detail': ' '.join(command)}) timing.begin('content_check') try: from .operation_queue import native_channel with native_channel(read_only=True): exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent) finally: timing.end('content_check') combined = '\n'.join(part for part in (stdout, stderr) if part) report = contents_report_path(request.lmc_file, request.block_name) try: report.parent.mkdir(parents=True, exist_ok=True) report.write_text(combined, encoding='utf-8') except Exception as error: stderr = f'{stderr}\n无法写入内容检查报告 {report}: {error}' status = classify_output(exit_code, combined, timed_out=timed_out) if status != 'PASS': # A contents failure or an unusable DMS must block before touching the PCB. return finish(VerifyResult(status if status in ('FAIL', 'TIMEOUT') else 'BLOCKED', stderr=combined.strip() or f'内容检查结果未知(退出码 {exit_code})', stdout=stdout, report_path=report, content_check_exit_code=exit_code, safe_to_continue=not timed_out)) if progress: _notify_progress(progress, {'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'}) # Sampling cannot establish that the whole native port range is unusable. from .mentor_adapter import icdb_ports_bindable check_ports = port_check or icdb_ports_bindable ports_ok, ports_detail = check_ports() if not ports_ok: if progress: _notify_progress(progress, {'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail}) if request.strict_port_check: return finish(VerifyResult('BLOCKED', stderr=ports_detail)) job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name, 'diagnostic_only': request.diagnostic_only, 'supervisor_pid': os.getpid(), 'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds, 'mentor_bin': str(request.mentor_bin or ''), 'mapped_drive': str(request.mapped_drive or ''), 'mapped_remote_root': str(request.mapped_remote_root or ''), 'interactive_input': bool(request.interactive_input), 'native_lm_verify': bool(request.native_lm_verify), 'desktop': 'CbbVerify_' + uuid4().hex[:12], 'cancel_signal': str(root/'cancel-worker.signal'), 'start_gate': str(root/'start-worker.signal')} job.update(operation_id=root.name, operation='verify', library=str(request.lmc_file), block=request.block_name, session_record_path=str(root/'session.json'), start_gate_protocol='event-v1') from .session_record import process_identity job['supervisor_identity'] = process_identity(os.getpid()) consent = None job_path = root/'request.json' try: if request.post_repair_proof is not None: from .post_repair_consent import PostRepairProof, ConsentController, exact_prompt if not isinstance(request.post_repair_proof, PostRepairProof): return finish(VerifyResult('BLOCKED', stderr='POST_REPAIR_PROOF_REQUIRED')) job['post_repair_consent'] = request.post_repair_proof.for_job(job) consent = ConsentController(request.post_repair_proof, job) job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8') except Exception as error: return finish(VerifyResult('FAIL', stderr=str(error))) events = root/'events.jsonl' prewrite_eligible = not (request.diagnostic_only or request.interactive_input or request.native_lm_verify or consent) def startup_phase(stage,**facts): record={k:job[k] for k in ('operation_id','project','pcb','desktop')} with events.open('a',encoding='utf-8') as stream: stream.write(json.dumps(dict(stage=stage,**record,**facts))+'\n') def before_dialog_action(snapshot,action): name=action.get('action') if isinstance(action,dict) else action if name not in ('dismiss_tip','accept_license','decline_database_log'): startup_phase('startup_supervisor_write_possible',action=name,pid=snapshot.pid,hwnd=snapshot.hwnd) offset = 0 target_pid = None diagnostic_closing = False launcher_pid = None lm_pid = None verify_invoked = False lm_dispatch_deadline = None invoke_returned = False pcb_baseline = set() last_observation = 0.0 last_post_verify_observation = {} from .mentor_adapter import ActivityDeadline, pcb_startup_timeout_seconds from .desktop_worker import DesktopWorker from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend from .dialog_models import DialogPhase from .operation_queue import TaskHandoff, check_active, native_channel check_active() worker_launch_attempted = False native = native_channel() phase_entered = False def end_native_phase(): nonlocal phase_entered if phase_entered: native.__exit__(None, None, None) phase_entered = False try: if prewrite_eligible:startup_phase('startup_supervisor_no_write') timing.begin('worker_setup') with (root/'worker.log').open('w', encoding='utf-8') as log: check_active() worker_launch_attempted = True if request.interactive_input: process = DesktopWorker( worker_command(job_path), None, interactive_input=True) else: process = DesktopWorker(worker_command(job_path), job['desktop']) if hasattr(process, 'bind_cancellation'): process.bind_cancellation(job['cancel_signal']) if hasattr(process, 'bind_session'): process.bind_session(job) deadline = ActivityDeadline(request.timeout_seconds) last_activity_key = None phase = DialogPhase.PCB_STARTING restore_controller = None open_controller = None dialog_agent = None latest_snapshots = () if dialog_supervisor_factory is None: if hasattr(process, 'start_companion'): from .dialog_agent import DialogAgentClient from .dialog_rules import default_rules_path, load_rules dialog_agent = DialogAgentClient.start(process, root/'dialog-agent', stop=stop) dialog_supervisor = DialogSupervisor( AgentDialogBackend(dialog_agent, before_action=before_dialog_action if prewrite_eligible else None, pids=lambda: { int(pid) for pid in (target_pid, lm_pid, launcher_pid) if pid}), rules=load_rules(default_rules_path(), diagnostic_only=request.diagnostic_only), ) else: dialog_supervisor = DialogSupervisor(Win32DialogBackend( lambda: ({value for value in (target_pid, lm_pid) if value is not None}), desktop=process.desktop,before_action=before_dialog_action if prewrite_eligible else None, )) else: dialog_supervisor = dialog_supervisor_factory( desktop=process.desktop, pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}), ) if stop is not None and stop.is_set(): raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB') check_active() try: native.__enter__() phase_entered = True except RuntimeError as error: return finish(VerifyResult('BLOCKED', stderr=str(error), safe_to_continue=False, error_category='NATIVE_CHANNEL_BUSY')) from .operation_queue import commit_worker_start commit_worker_start(process, job) while process.poll() is None: if stop is not None and stop.is_set(): save_markers = timing._save_markers detail = ('保存已返回,关闭待确认' if 'SAVE_OK' in save_markers and 'CLOSE_OK' not in save_markers else '已请求停止,等待当前保存返回' if phase == DialogPhase.SAVE else '已请求停止,等待当前原生操作返回') _notify_progress(progress, {'stage': 'save_stop_requested', 'block': request.block_name, 'detail': detail}) from .failed_session import cancelled_prewrite_startup _stop_worker(process, wait_for_native_write=False if request.diagnostic_only else lambda:cancelled_prewrite_startup(job,target_pid) is None) return finish(VerifyResult('CANCELLED', stderr='已取消当前校验。', safe_to_continue=False, error_stage=phase.value)) if events.exists(): with events.open(encoding='utf-8') as stream: stream.seek(offset) while line := stream.readline(): try: event = json.loads(line) except json.JSONDecodeError: break timing.record(event) if _diagnostic_close_event(job, event, target_pid): _acknowledge_diagnostic_close(job, event, stop) diagnostic_closing = True if event.get('pcb_pid'): target_pid = event['pcb_pid'] if event.get('launcher_pid'): launcher_pid = event['launcher_pid'] if event.get('lm_pid'): lm_pid = event['lm_pid'] stage = event.get('stage') if stage == 'lm_launch' and lm_dispatch_deadline is None: from .mentor_adapter import lm_dispatch_timeout_seconds lm_dispatch_deadline = (event.get('monotonic_seconds') or time.monotonic()) + lm_dispatch_timeout_seconds(request.timeout_seconds) if stage == 'verify_invoke': verify_invoked = True if stage == 'verify_invoke_returned': invoke_returned = True if stage == 'pcb_baseline': pcb_baseline = set(event.get('pids', ())) phase = phase_for_event(event, phase) activity_key = (stage, json.dumps(event.get('statuses'), sort_keys=True), json.dumps(event.get('markers'), sort_keys=True), event.get('action'), event.get('detail')) # Repeated integration_status snapshots are normal # polling, not progress. Only a changed state or # a new stage/marker extends the inactivity window. if activity_key != last_activity_key: if stage in ('lm_started', 'lm_ready', 'verify_invoke', 'verify_invoke_returned', 'lm_session_ready', 'pcb_started', 'pcb_worker_started'): deadline.timeout_seconds = pcb_startup_timeout_seconds( request.timeout_seconds) elif stage == 'identity_verified': deadline.timeout_seconds = max( 0.1, float(request.timeout_seconds)) deadline.touch() last_activity_key = activity_key if progress: _notify_progress(progress, event) offset = stream.tell() if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5: from .post_verify_diagnostics import observe_post_verify last_observation = time.monotonic() try: observation = observe_post_verify(process.desktop, pcb, pcb_baseline, lm_pid) except Exception as error: observation = {'error': str(error)} last_post_verify_observation = observation if diagnostic_closing: # The exact diagnostic worker owns its bounded close/No sequence. from .dialog_supervisor import DialogPollResult dialog_result = DialogPollResult(activity=False) elif dialog_agent is not None and target_pid is None: # The Library Manager Verify route starts its own window # before it launches Expedition PCB. Do not feed that # large MFC main window to the PCB dialog probe; it can # exhaust the UIA observation deadline before Verify has # even produced a target process. from .dialog_supervisor import DialogPollResult dialog_result = DialogPollResult(activity=False) elif dialog_agent is not None: dialog_result = dialog_supervisor.poll(phase) elif dialog_supervisor_factory is not None: try: dialog_result = dialog_supervisor.poll(phase) except TypeError: # Compatibility for injected legacy test supervisors only. dialog_result = dialog_supervisor.poll() else: dialog_result = dialog_supervisor.poll() if not diagnostic_closing: intent = _close_intent_after_poll(job, events, offset, target_pid) if intent is not None: _acknowledge_diagnostic_close(job, intent, stop) diagnostic_closing = True from .dialog_supervisor import DialogPollResult dialog_result = DialogPollResult(activity=False) if (version_prompt is not None and consent is None and dialog_agent is not None and not request.diagnostic_only and not request.native_lm_verify and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED' and dialog_result.snapshot is not None): from .pcb_open_consent import OpenController, exact_prompt if exact_prompt(dialog_result.snapshot): if open_controller is None: open_controller = OpenController(job, auto_confirm=( request.auto_confirm_pcb_open and not request.maintenance)) dialog_result = open_controller.handle(dialog_result.snapshot, dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid, show=version_prompt, perform=dialog_supervisor.backend.perform) if (consent is not None and dialog_agent is not None and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED' and dialog_result.snapshot is not None and exact_prompt(dialog_result.snapshot)): consent_snapshot = dialog_result.snapshot if consent_snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}: dialog_result = consent.handle(consent_snapshot, dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid, companion_pids=tuple(process.companion_pids), perform=dialog_supervisor.backend.perform) if dialog_result.error_code == 'PCB_VERSION_CHOICE_REQUIRED' and dialog_result.snapshot is not None: from .dialog_rules import archive_version_choice choice_path = archive_version_choice(dialog_result.snapshot, root) dialog_result = replace(dialog_result, detail=dialog_result.detail + f'\n版本候选与原生窗口证据:{choice_path}') if (version_prompt is not None and not request.diagnostic_only and dialog_agent is not None and dialog_result.snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}): from .pcb_restore_prompt import RestoreController if restore_controller is None: restore_controller = RestoreController(job, user_open=consent is None) dialog_result = restore_controller.handle(dialog_result.snapshot, dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid, companion_pids=tuple(process.companion_pids), show=version_prompt, perform=dialog_supervisor.backend.perform) # Normal progress stays in memory. Only actions/errors and the # last failure snapshots are persisted, not every elapsed tick. latest_snapshots = dialog_result.observed_snapshots if dialog_result.snapshot is not None and ( dialog_result.action_sent or dialog_result.error_code): snapshot_record = (dialog_result.snapshot.to_record() if hasattr(dialog_result.snapshot, 'to_record') else dialog_result.snapshot.as_log_record()) record = { 'time': time.strftime('%Y-%m-%d %H:%M:%S'), **snapshot_record, 'action': dialog_result.action, 'rule_id': dialog_result.rule_id, 'channel': dialog_result.channel, 'accepted_control': list(dialog_result.accepted_controls), 'error_code': dialog_result.error_code, 'detail': dialog_result.detail, } with (root/'dialogs.jsonl').open('a', encoding='utf-8') as stream: stream.write(json.dumps(record, ensure_ascii=False) + '\n') if dialog_result.activity: deadline.touch() if dialog_result.error_code: _write_failure_windows(root, latest_snapshots) _write_process_tree(root, process, target_pid) # A failed database load has no document to close. Give the # worker time to normally close its owned empty PCB before # recovery checks for remaining native database clients. _stop_worker(process, timeout=40) return finish(VerifyResult( 'CANCELLED' if dialog_result.error_code == 'USER_CANCELLED' else 'SKIPPED' if dialog_result.error_code == 'PCB_OPEN_SKIPPED' else 'FAIL', stderr=f'{dialog_result.error_code}: {dialog_result.detail}', error_category='PCB_OPEN_SKIPPED' if dialog_result.error_code == 'PCB_OPEN_SKIPPED' else '', error_stage=phase.value, safe_to_continue=False, )) if deadline.expired() or (lm_dispatch_deadline is not None and target_pid is None and time.monotonic() >= lm_dispatch_deadline): _write_failure_windows(root, latest_snapshots) # Stop automation without killing PCB/iCDB during a write. _write_process_tree(root, process, target_pid) if last_post_verify_observation: (root/'post_verify_observations.jsonl').write_text(json.dumps({ 'time': time.strftime('%Y-%m-%d %H:%M:%S'), 'invoke_returned': invoke_returned, **last_post_verify_observation, }, ensure_ascii=False) + '\n', encoding='utf-8') dispatch_failure = classify_verify_dispatch( verify_invoked=verify_invoked, invoke_returned=invoke_returned, target_pid=target_pid, observation=last_post_verify_observation, ) _stop_worker(process, wait_for_native_write=not request.diagnostic_only) if dispatch_failure is not None: status, detail = dispatch_failure if progress: _notify_progress(progress, { 'stage': 'verify_dispatch_failed', 'block': request.block_name, 'detail': detail, 'observation': last_post_verify_observation, }) return finish(VerifyResult(status, stderr=detail, safe_to_continue=False)) return finish(VerifyResult('TIMEOUT', stderr='操作超时,当前项未完成。', error_stage=phase.value, safe_to_continue=False)) time.sleep(.2) end_native_phase() # Exact worker exit precedes all optional outcome/evidence I/O. worker_result = root/'outcome.json' if not worker_result.exists(): return finish(VerifyResult('FAIL', stderr=f'执行器异常退出 ({process.returncode}),见 worker.log', safe_to_continue=False)) outcome = json.loads(worker_result.read_text(encoding='utf-8')) if outcome.get('status') == 'PASS' and process.returncode != 0: return finish(VerifyResult('FAIL', stderr='执行器异常退出,不能确认成功', safe_to_continue=False)) result = VerifyResult(**outcome) except Exception as error: if 'process' in locals() and process.poll() is None: _stop_worker(process, wait_for_native_write=not request.diagnostic_only) launch_release = getattr(error, 'worker_launch_release', None) if 'process' not in locals() else None if not worker_launch_attempted: launch_release = {'process_created': False, 'worker_exited': True, 'safe_to_continue': True, 'remaining_pids': []} no_launch = (isinstance(launch_release, dict) and launch_release.get('process_created') is False and launch_release.get('worker_exited') is True and launch_release.get('safe_to_continue') is True and launch_release.get('remaining_pids') == []) if no_launch: _write_optional_evidence(root/'session-release.json', { **launch_release, 'block': request.block_name, 'library': str(request.lmc_file), 'worker_pid': None}) status = 'CANCELLED' if stop is not None and stop.is_set() else 'FAIL' return finish(VerifyResult(status, stderr=str(error), safe_to_continue=no_launch)) finally: close_errors = [] worker_exited = False observer_started = False automation_release = {'safe_to_continue': False} if 'process' not in locals() or process.poll() is not None or getattr(process, 'session_record_path', None): # A gated worker owns its writer barrier and inherited sidecar. # Parent admission must not block controls needed for its exit. end_native_phase() if 'dialog_agent' in locals() and dialog_agent is not None: try: dialog_agent.close() except Exception as error: close_errors.append(str(error)) if 'process' in locals(): try: worker_exited = process.poll() is not None if not worker_exited: worker_exited = _stop_worker(process, wait_for_native_write=not request.diagnostic_only) if hasattr(process, 'automation_release_state'): automation_release = process.automation_release_state() else: automation_release = {'safe_to_continue': worker_exited, 'remaining_pids': []} if not worker_exited and getattr(process, 'session_record_path', None): from .session_record import handoff_session handoff_session(process.session_record_path, evidence_root=root, process=process, progress=lambda state: _notify_progress(progress, {'stage': 'session_observation', 'block': request.block_name, **state})) observer_started = True _notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name, 'detail': '已转后台观察,可查看状态'}) from .operation_queue import hold_native_handoff hold_native_handoff(process, process.session_record_path) except TaskHandoff as error: handoff_result = _handoff_verify_result(error) if handoff_result.evidence_path: _write_optional_evidence(handoff_result.evidence_path, {'block_name': request.block_name, **asdict(handoff_result)}) raise except Exception as error: close_errors.append(str(error)) if 'process' in locals(): try: if events.is_file(): with events.open(encoding='utf-8') as stream: stream.seek(offset) for line in stream: try: event = json.loads(line) timing.record(event) if event.get('pcb_pid'): target_pid = event['pcb_pid'] if event.get('launcher_pid'): launcher_pid = event['launcher_pid'] if event.get('lm_pid'): lm_pid = event['lm_pid'] if event.get('stage') == 'pcb_baseline': pcb_baseline = set(event.get('pids', ())) except (json.JSONDecodeError, ValueError): continue offset = stream.tell() release_result = _cleanup_session_processes( root, lm_pid=lm_pid, target_pid=target_pid, launcher_pid=launcher_pid, pcb=pcb, pcb_baseline=pcb_baseline, progress=progress, ) except Exception as error: release_result = {'safe_to_continue': False, 'error': str(error)} try: from .icdb_ownership import unreleased_addresses addresses = unreleased_addresses(request.lmc_file, project, pcb) if addresses: release_result = {**(release_result or {}), 'icdb_addresses': addresses} except OSError as error: close_errors.append(str(error)) if close_errors: release_result = {**(release_result or {}), 'safe_to_continue': False, 'close_errors': close_errors} if not worker_exited or not automation_release.get('safe_to_continue'): release_result = {**(release_result or {}), 'safe_to_continue': False, 'automation_release': automation_release, 'worker_exited': worker_exited} release_result['remaining_pids'] = sorted(set(release_result.get('remaining_pids', [])) | set(automation_release.get('remaining_pids', []))) if worker_exited: try: if (release_result.get('safe_to_continue') is not True and getattr(process, 'session_record_path', None)): if not observer_started: from .session_record import handoff_session # Retain the exact worker handle until registered # native clients exit and the observer obtains proof. handoff_session(process.session_record_path, evidence_root=root, process=process, progress=lambda state: _notify_progress(progress, {'stage': 'session_observation', 'block': request.block_name, **state})) observer_started = True _notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name, 'detail': '工作进程已退出,原生会话继续后台观察'}) else: process.close() except Exception as error: release_result = {**release_result, 'safe_to_continue': False, 'close_errors': [*close_errors, str(error)]} try: from .operation_lifecycle import classify_save_state save_lines = ((root/'save.log').read_text('utf-8', errors='replace').splitlines() if (root/'save.log').is_file() else []) _write_optional_evidence(root/'write-outcome.json', classify_save_state(save_lines, helper_exited=(False if automation_release.get('remaining_pids') else True if worker_exited and automation_release.get('safe_to_continue') else None), native_released=release_result.get('safe_to_continue') is True)) (root/'session-release.json').write_text(json.dumps({ **(release_result or {}), 'block': request.block_name, 'library': str(request.lmc_file), 'worker_exited': worker_exited, 'worker_pid': getattr(process, 'pid', None)}, ensure_ascii=False, default=str, indent=2), encoding='utf-8') if worker_exited and release_result.get('safe_to_continue') is True and (root/'session.json').is_file(): from .session_record import write_session_record write_session_record(root/'observation-supervisor.json', { 'operation_id': job['operation_id'], 'status': 'FINISHED', 'native_release_confirmed': True, 'worker_exited': True}) except OSError: pass if not release_result or not release_result.get('safe_to_continue'): if 'result' in locals(): result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False, stderr=(result.stderr + '\nPCB_RELEASE_UNCONFIRMED: 未确认原生会话退出').strip()) recovery = {'desktop': job['desktop'], 'project': str(project), 'pcb': str(pcb), 'release': release_result, 'action': '停止批次;检查原生会话及数据库;不要删除锁文件或自动恢复。'} try: (root/'recovery-required.json').write_text( json.dumps(recovery, ensure_ascii=False, indent=2), encoding='utf-8') except OSError: pass # Evidence failure must not prevent mutex release. # Drain the timing tail even when the worker exited before a poll. # This is optional telemetry: an unreadable log must not mask failure # or prevent releasing the serial-operation mutex. try: if events.is_file(): with events.open(encoding='utf-8') as stream: stream.seek(offset) for line in stream: try: timing.record(json.loads(line)) except (json.JSONDecodeError, ValueError): continue except OSError: pass finally: end_native_phase() if result.status == 'PASS' and not request.diagnostic_only: timing.begin('library_flag') bridge_records = [] def record_attempt(record): bridge_records.append(record) if progress: _notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record}) try: with native_channel(read_only=True): probe = read_library_flag_with_retry( flag_reader or read_library_flag_if_available, Path(request.lmc_file), request.block_name, on_attempt=record_attempt) result = apply_post_verify_status(result, probe) result = replace(result, bridge_attempts=tuple(bridge_records)) _write_optional_evidence(root/'bridge-status.json', bridge_records) if progress: _notify_progress(progress, {'stage': 'library_flag', 'block': request.block_name, 'library_flag': result.library_flag, 'detail': result.library_flag_detail}) finally: timing.end('library_flag') return finish(result) def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None, context=None): """Run serially; isolate failures and skip writes to resources still retained.""" if max_workers != 1: raise ValueError('Only serial Verify is supported for the shared Library Manager session') if context is not None: stop = stop if stop is not None else context.stop progress = context.guard_callback(progress) if progress is not None else None if verifier is None: verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available, stop=stop, context=context) # noqa: E731 results = [] queue_stop = False retained_libraries = {} for request in requests: from .operation_queue import request_library_keys library_keys = request_library_keys(request, context) for index in {retained_libraries[key][1] for key in library_keys.intersection(retained_libraries)}: previous = next(value[0] for value in retained_libraries.values() if value[1] == index) if queue_stop or stop is not None and stop.is_set(): break if not results[index].evidence_path: continue from .operation_lifecycle import recheck_failed_release refreshed = recheck_failed_release(previous.lmc_file, previous.block_name, results[index], stop=stop) if refreshed.safe_to_continue: results[index] = refreshed retained_libraries = {k:v for k,v in retained_libraries.items() if v[1] != index} if (queue_stop or stop is not None and stop.is_set() or context is not None and context.stop.is_set()): result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB') elif library_keys.intersection(retained_libraries): previous, index = retained_libraries[next(iter(library_keys.intersection(retained_libraries)))] result = VerifyResult('SKIPPED', error_category='PREVIOUS_SESSION_UNRELEASED', error_stage='admission', stderr=f'前项 {getattr(previous, "block_name", previous)} 的同库会话尚未释放,当前项未启动。') else: try: def execute(): result = verifier(request) if not result.safe_to_continue: from .operation_lifecycle import recheck_failed_release result = recheck_failed_release(request.lmc_file, request.block_name, result, stop=stop) return result result = item_executor(request, execute) if item_executor is not None else execute() except Exception as error: from .operation_queue import QueueCancelled, QueueUnavailable, TaskHandoff # An unexpected exception cannot establish database safety. result = (VerifyResult('SKIPPED', stderr=str(error), error_category='NATIVE_SESSION_PENDING', error_stage='admission') if isinstance(error, QueueUnavailable) else _handoff_verify_result(error) if isinstance(error, TaskHandoff) else VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL', stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled))) results.append(result) # Never start another writer after an unconfirmed native shutdown. if not result.safe_to_continue: retained_libraries.update({key:(request, len(results)-1) for key in library_keys}) queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'} if queue_stop and stop is not None: stop.set() if progress: _notify_progress(progress, request.block_name, result) return results def failure_report_path(batch_report): batch_report = Path(batch_report) return batch_report.with_name(batch_report.name.replace('batch-', 'failures-', 1)) def _report_field(value): # One CBB per row; full native multi-line text remains in evidence files. return str(value).replace('\r', r'\r').replace('\n', r'\n').replace('\t', r'\t') def write_failure_report(requests, results, root, *, path=None): requests, results = tuple(requests), tuple(results) if len(requests) != len(results): raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告') root = Path(root); root.mkdir(parents=True, exist_ok=True) path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log') failed = [(req, result) for req, result in zip(requests, results) if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'} and not (result.status in {'CHECKED', 'REPAIRED'} and result.safe_to_continue)] passed = sum(result.status == 'PASS' for result in results) checked = sum(result.status == 'CHECKED' and result.safe_to_continue for result in results) repaired = sum(result.status == 'REPAIRED' and result.safe_to_continue for result in results) skipped = len(results)-passed-len(failed)-checked-repaired lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped} 仅检查未修复: {checked} ICDB已修复待校验: {repaired}', 'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)'] if requests and all(req.maintenance for req in requests): from .icdb_maintenance import maintenance_counts counts = maintenance_counts(results) lines[:2] = ['ICDB 检查异常清单', f'总数: {len(results)} 未发现异常: {checked} 发现问题: {counts["issues_found"]} ' f'占用: {counts["in_use"]} 无法确认: {counts["unconfirmed"]} ' f'取消: {counts["cancelled"]} 未执行: {counts["skipped"]}'] for req, result in failed: timings = ' '.join(f'{key}={value:.3f}s' + ('(unfinished)' if key in result.incomplete_timing_phases else '') for key, value in result.timings_seconds.items()) lines.append(f'{req.block_name}\t{result.status}\t{result.error_category or classify_failure_category(result.stderr)}\t' f'{result.error_stage or "unknown"}\t{result.library_flag}\t{_report_field(result.stderr or result.stdout.strip()[:200])}\t{result.evidence_path or ""}\t{timings}') if not failed: lines.append('本批次没有失败项。') path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig') return path def write_batch_report(requests, results, root): requests, results = tuple(requests), tuple(results) if len(requests) != len(results): raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告') root = Path(root); root.mkdir(parents=True, exist_ok=True) path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log') lines = ['CBB Verify 批量结果', '流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)'] if requests and all(req.maintenance for req in requests): lines = ['ICDB 批量检查结果', '流程: 原生 PCB 打开检查;不执行 Repair、Restore 或 CBB 校验。'] for req, result in zip(requests, results): detail = _report_field(result.stderr or result.stdout.strip()[:200]) seconds = ' '.join(f'{key}={value:.3f}s' + ('(unfinished)' if key in result.incomplete_timing_phases else '') for key, value in result.timings_seconds.items()) lines.append(f'{req.block_name}\t{result.status}\t{result.library_flag}\t{detail}\t{result.evidence_path or ""}\t{seconds}') path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig') write_failure_report(requests, results, root, path=failure_report_path(path)) return path