Files
cbb-refresh-verify-795/src/cbb_refresh_verify/verify_runner.py
T

1635 lines
92 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Serial Verify supervisor.
Each CBB gets one isolated, bounded automation worker on a private Windows
desktop. The worker reproduces the observed refresh flow: invoke the CBB's
native Library Manager Verify action, bind the resulting ReusableCircuit PCB,
verify the document identity, run Project Integration, save the Reusable Block
cell and close. Optionally the official
``ReuseBlockImporter.exe -chkContents`` check runs first when a LibSpec and a
DMS login configuration are supplied.
Nothing here injects mouse or keyboard input or edits the central library. The
native menu invocation happens on the worker's private desktop.
"""
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
import json
import logging
import os
import platform
import re
import struct
import subprocess
import sys
import time
from uuid import uuid4
from .network_path import absolute_preserving_drive
from .runtime_timing import RunTimings
from .defaults import DEFAULT_TIMEOUT_SECONDS
SUCCESS_TEXT = re.compile(r'content checking completed successfully', re.IGNORECASE)
ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE)
INVALID_NAME_CHARS = '\\/:*?"<>|'
def _notify_progress(callback, *args):
"""A failed observer cannot change native execution or release facts."""
if callback is not None:
try:
callback(*args)
except Exception:
logging.getLogger(__name__).warning('Progress notification failed', exc_info=True)
def _write_optional_evidence(path, value):
try:
path.write_text(json.dumps(value, ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except OSError:
logging.getLogger(__name__).warning('Could not write diagnostic evidence %s', path, exc_info=True)
@dataclass(frozen=True)
class VerifyRequest:
lmc_file: Path
block_name: str
evidence_dir: Path
libspec: str = ''
dms_login_config: str = ''
mentor_bin: str = ''
# PCB startup on EE7.9 includes iCDB recovery and COM registration. The
# previous 30-second UI default was shorter than the observed startup
# window on this machine and caused a false timeout before document attach.
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
logging_context: object | None = None
# The production default stays on an isolated desktop. The explicit
# interactive mode is reserved for a real-input compatibility test on
# machines where EE7.9 cannot dispatch the tree menu on a private desktop.
interactive_input: bool = False
# The native Library Manager menu route is opt-in until a machine's menu
# dispatch has been proven. The flag is serialized to the worker so a
# Win10 compatibility run can select it explicitly.
native_lm_verify: bool = False
# A reserved iCDB range is a host-wide prerequisite failure. Do not
# launch a PCB that is guaranteed to time out; batch orchestration will
# record this item and continue with the next one.
strict_port_check: bool = False
# A mapped drive can be invisible to an elevated/private worker on Win10.
# Preserve the parent session's mapping identity so the worker can restore
# it before launching Library Manager/PCB.
mapped_drive: str = ''
mapped_remote_root: str = ''
auto_repair: bool = False
maintenance: bool = False
post_repair_proof: object | None = field(default=None, repr=False, compare=False)
diagnostic_only: bool = False
auto_confirm_pcb_open: bool = False
@dataclass(frozen=True)
class VerifyResult:
status: str
stderr: str = ''
stdout: str = ''
evidence_path: Path | None = None
safe_to_continue: bool = True
library_flag: str = 'UNKNOWN'
library_flag_detail: str = ''
report_path: Path | None = None
content_check_exit_code: int | None = None
error_category: str = ""
error_stage: str = ""
bridge_attempts: tuple[dict, ...] = ()
timings_seconds: dict[str, float] = field(default_factory=dict)
incomplete_timing_phases: tuple[str, ...] = ()
recovery_attempts: tuple[dict, ...] = ()
release_proof: dict | None = None
write_outcome: dict | None = None
session_record_path: Path | None = None
def resolve_importer(mentor_bin: str | None = None) -> Path:
"""Configured bin first, then the EE7.9.5 environment defaults."""
candidates = []
if mentor_bin:
configured = Path(mentor_bin)
if configured.is_file() and configured.name.casefold() == 'reuseblockimporter.exe':
candidates.append(configured.parent)
else:
candidates.append(configured)
if configured.name.casefold() == 'sdd_home':
candidates.append(configured / 'lm' / 'win32' / 'bin')
from .mentor_paths import discover_sdd_home
discovered = discover_sdd_home(mentor_bin)
if discovered is not None:
candidates.append(discovered / 'lm' / 'win32' / 'bin')
for variable in ('SDD_LM_BIN', 'SDD_HOME'):
value = os.environ.get(variable)
if not value:
continue
root = Path(value)
candidates.append(root if root.name.lower() == 'bin' else root/'lm'/'win32'/'bin')
candidates.append(Path(r'D:\Mentor\7.9.5EE\SDD_HOME\lm\win32\bin'))
for candidate in candidates:
executable = candidate/'ReuseBlockImporter.exe'
if executable.is_file():
return executable
raise ValueError('ReuseBlockImporter.exe 未找到;请设置 SDD_LM_BIN')
def build_content_check_command(request, executable) -> tuple[str, ...]:
"""The documented Logical-Physical contents check invocation."""
return (str(executable), '-lmcFile', str(request.lmc_file), '-blockName', request.block_name,
'-logicalType', 'LP', '-chkContents', '-libSpec', str(request.libspec),
'-dmsLoginConfig', str(request.dms_login_config))
def contents_report_path(lmc_file, block_name) -> Path:
safe_name = re.sub(f'[{re.escape(INVALID_NAME_CHARS)}]', '_', str(block_name))
return Path(lmc_file).parent/'LogFiles'/f'restorerb_RULP_{safe_name}_contents.log'
def classify_output(exit_code: int | None, output: str, *, timed_out: bool = False) -> str:
"""Never invent success: PASS needs the success text and no error marker."""
if timed_out:
return 'TIMEOUT'
from .icdb_recovery import parse_session_loss
if parse_session_loss(output):
return 'FAIL'
diagnostic_text = re.sub(r'\b0\s+errors?\b', '', output, flags=re.I)
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(diagnostic_text):
return 'PASS'
if ERROR_TEXT.search(output):
return 'FAIL'
if exit_code not in (None, 0):
return 'FAIL'
return 'UNKNOWN'
def preflight(request):
"""Validate the library entry and its ReusableCircuit sources before launch."""
library = absolute_preserving_drive(request.lmc_file)
name = request.block_name
if not name or name in ('.', '..') or any(character in name for character in INVALID_NAME_CHARS):
raise ValueError('CBB 名称非法')
if not library.is_file():
raise ValueError(f'中心库不存在: {library}')
work = library.parent/'ReusableBlocksLP'/name/'Work'
projects = list(work.glob('*.prj'))
if len(projects) != 1:
raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ')
project = absolute_preserving_drive(projects[0])
from .backup_recovery import _project_text
content = _project_text(project)
refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M)
if not refs:
raise ValueError(f'{name}: 工程没有 PCBDesignPath')
pcb = absolute_preserving_drive(project.parent/refs[0])
if not pcb.is_relative_to(absolute_preserving_drive(work)):
raise ValueError(f'{name}: PCB 引用越出当前 CBB,停止操作: {pcb}')
real_work = work.resolve()
expected_work = library.parent.resolve() / 'ReusableBlocksLP' / name / 'Work'
if os.path.normcase(str(real_work)) != os.path.normcase(str(expected_work)):
raise ValueError(f'{name}: Work 路径通过链接指向其他工程,停止操作')
for candidate in (project, pcb, pcb.parent/'pcb.prj', pcb.parent/'Layout',
pcb.parent/'Layout'/'LayoutDB.lyt', work/'database', work/'CES'):
if not candidate.resolve().is_relative_to(real_work):
raise ValueError(f'{name}: 文件或目录链接越出当前 CBB,停止操作: {candidate}')
if not pcb.is_file() or not (pcb.parent/'Layout'/'LayoutDB.lyt').is_file():
raise ValueError(f'{name}: 缺少 PCB 或布局数据库: {pcb}')
return project, pcb
def preflight_skip_reason(lmc_file, block_name):
"""Return a user-facing skip reason for an import-incomplete CBB.
This is intentionally narrower than ``preflight``: only local source
structure problems are excluded before a batch starts. Runtime failures
such as ports, DMS, COM, or Project Integration remain normal per-item
failures and are still attempted when the user queues them.
"""
library = absolute_preserving_drive(lmc_file)
if not library.is_file():
return ''
work = library.parent / 'ReusableBlocksLP' / str(block_name) / 'Work'
if not work.is_dir():
return f'{block_name}: 缺少 Work 工程目录'
request = VerifyRequest(library, str(block_name), library.parent / '.cbb-preflight-evidence')
try:
preflight(request)
except Exception as error:
detail = str(error)
if any(marker in detail for marker in ('缺少或无法唯一确定 Work 下的工程 PRJ',
'工程没有 PCBDesignPath',
'缺少 PCB 或布局数据库')):
return detail
return ''
def _run_process(command, timeout_seconds, cwd, *, evidence_root=None):
from .content_check import run_content_check
return run_content_check(command, timeout_seconds, cwd, evidence_root=evidence_root)
def _stop_worker(process, *, timeout=5, wait_for_native_write=True):
"""Request stop without interrupting a possibly unreturned native call."""
try:
if hasattr(process, 'request_stop'):
process.request_stop()
except Exception:
pass
try:
if getattr(process, '_stop_wait_complete', False):
return process.poll() is not None
process.wait(timeout=timeout)
except Exception:
pending = getattr(process, 'native_write_pending', lambda: False)()
wait_required = wait_for_native_write() if callable(wait_for_native_write) else wait_for_native_write
if wait_required and pending is not False and not getattr(process, '_stop_wait_complete', False):
from .defaults import NATIVE_WRITE_STOP_WAIT_SECONDS
try:
process.wait(timeout=max(0, NATIVE_WRITE_STOP_WAIT_SECONDS - timeout))
except Exception:
pass
process._stop_wait_complete = True
try:
return process.poll() is not None
except Exception:
return False
def _normalize_bridge_result(probe):
from .status_bridge import BridgeResult
if not isinstance(probe, BridgeResult):
detail = str(probe[1]) if isinstance(probe, tuple) and len(probe) > 1 else str(probe)
return BridgeResult('MALFORMED', '', None,
'BRIDGE_RESULT_INVALID: 非权威桥接器结果;' + detail)
if probe.status in {'READY', 'PENDING'} and (
probe.source != 'IsVerifiedReuseBlock'
or probe.raw_status != (1 if probe.status == 'READY' else 0)):
return BridgeResult('MALFORMED', probe.source, probe.raw_status,
'BRIDGE_RESULT_INVALID: status/source/raw_status 不一致')
return probe
def read_library_flag_with_retry(reader, library, block_name, *, attempts=3,
delays=(0.0, 3.0, 5.0), sleep=None, on_attempt=None):
"""Bounded read-only bridge retries; never repeat Verify or promote PENDING."""
from .status_bridge import BridgeResult
sleep = sleep or time.sleep
last = BridgeResult('UNAVAILABLE', '', None, '状态桥没有返回结果')
for index in range(max(1, int(attempts))):
if index > 0:
delay = delays[min(index, len(delays)-1)] if delays else 0
sleep(max(0.0, float(delay)))
started = time.monotonic()
try:
last = _normalize_bridge_result(reader(library, block_name))
except Exception as error:
last = BridgeResult('UNAVAILABLE', '', None, 'BRIDGE_UNAVAILABLE: ' + str(error))
if on_attempt:
on_attempt({'attempt': index + 1, 'status': last.status,
'source': last.source, 'raw_status': last.raw_status,
'detail': last.detail,
'query_seconds': round(time.monotonic()-started, 6)})
if last.status in {'READY', 'MALFORMED'}:
return last
return last
def classify_failure_category(detail):
text = str(detail or '')
lower = text.casefold()
if 'pcb_version_choice_required' in lower:
return 'PCB_VERSION_CHOICE_REQUIRED'
from .icdb_recovery import ERROR_CODE, parse_session_loss
if ERROR_CODE.casefold() in lower or parse_session_loss(text):
return ERROR_CODE
if 'back annotation is not allowed on verified (or once verified) reusable blocks.' in lower:
# Preserve the native integration failure even when shutdown adds a
# secondary release warning. This category does not grant permission
# to reimport or change safe_to_continue.
return 'CBB_REIMPORT_REQUIRED'
if any(code in lower for code in (
'pcb_release_unconfirmed', 'icdb_release_unconfirmed',
'icdb_session_unconfirmed', 'pcb_session_active', 'database_recovery_required')):
return 'DATABASE_SAFETY_STOP'
if 'tf_floatinglangbar_wndtitle' in lower or 'cicerouiwndframe' in lower:
return 'TOOL_FALSE_DIALOG'
if 'could not load job preferences db' in lower or 'unable to create local cell library' in lower:
return 'MENTOR_LOCAL_CELL_LIBRARY'
if ('database_load_failed' in lower
or 'problem encountered while opening one of the database files' in lower
or 'database load is being terminated' in lower):
return 'MENTOR_DATABASE_LOAD_FAILED'
if 'project integration error' in lower or 'project_integration_failed' in lower:
return 'MENTOR_PROJECT_INTEGRATION'
if 'save_reusable_block_failed' in lower or 'error saving the reusable block' in lower:
return 'MENTOR_SAVE_FAILED'
if 'design_server_failed' in lower or 'could not run server' in lower or 'could not bring it alive' in lower:
return 'ENVIRONMENT_DESIGN_SERVER'
if 'bridge_unavailable' in lower or '权威状态读取失败' in text:
return 'BRIDGE_UNAVAILABLE'
if 'bridge_result_invalid' in lower:
return 'BRIDGE_RESULT_INVALID'
if 'pending' in lower and 'isverifiedreuseblock' in lower:
return 'MENTOR_VERIFY_NOT_COMMITTED'
if 'cert' in lower and ('incomplete' in lower or '成果' in text):
return 'TOOL_CERT_DIAGNOSTIC'
if 'unknown_dialog' in lower or 'dialog' in lower:
return 'TOOL_DIALOG_SUPERVISOR'
if 'timeout' in lower or '超时' in text:
return 'TOOL_TIMEOUT'
return 'UNCLASSIFIED'
def read_library_flag_if_available(library, block_name):
"""Read the authoritative Mentor Verify state through the x86 bridge.
Unit tests replace this external boundary; production never accepts Cert
files or legacy tree readers as a substitute for this gold standard.
"""
from .status_bridge import StatusBridgeClient
return StatusBridgeClient(
env={'CBB_STATUS_BRIDGE_ALLOW_PRIVATE': '1'}
).query(Path(library), block_name)
def apply_post_verify_status(result, bridge_result):
"""Only the authoritative bridge can confirm READY, never a Cert/tuple/icon."""
if result.status != 'PASS':
return result # a pre-existing READY cannot erase a real execution error
bridge_result = _normalize_bridge_result(bridge_result)
if bridge_result.status == 'READY':
return replace(result, library_flag='GREEN', library_flag_detail=(
f'READY ({bridge_result.source}, raw={bridge_result.raw_status})'))
detail = bridge_result.detail or (
f'执行已完成,但权威状态仍为 {bridge_result.status} '
f'({bridge_result.source}, raw={bridge_result.raw_status})')
pending = bridge_result.status == 'PENDING'
return replace(result, status='FAIL',
stderr='\n'.join(part for part in (result.stderr, detail) if part),
library_flag='NOT_GREEN' if pending else 'UNKNOWN', library_flag_detail=detail,
error_category=('MENTOR_VERIFY_NOT_COMMITTED' if pending else
'BRIDGE_RESULT_INVALID' if bridge_result.status == 'MALFORMED' else 'BRIDGE_UNAVAILABLE'),
error_stage='library_flag', safe_to_continue=True)
def worker_command(request_path):
if getattr(sys, 'frozen', False):
return [sys.executable, '--verify-worker', str(request_path)]
return [sys.executable, '-m', 'cbb_refresh_verify', '--verify-worker', str(request_path)]
def phase_for_event(event, current=None):
from .dialog_models import DialogPhase
stage = str(event.get('stage', ''))
mapping = {
'lm_launch': DialogPhase.PCB_STARTING,
'lm_started': DialogPhase.PCB_STARTING,
'lm_ready': DialogPhase.PCB_STARTING,
'lm_session_ready': DialogPhase.PCB_STARTING,
'verify_invoke': DialogPhase.PCB_STARTING,
'verify_invoke_returned': DialogPhase.PCB_STARTING,
'pcb_launch': DialogPhase.PCB_STARTING,
'pcb_started': DialogPhase.PCB_STARTING,
'com_wait': DialogPhase.DOCUMENT_ATTACH,
'identity_wait': DialogPhase.DOCUMENT_ATTACH,
'identity_verified': DialogPhase.IDENTITY_CHECK,
'license_key_requested': DialogPhase.LICENSE_ACQUISITION,
'license_token_received': DialogPhase.LICENSE_ACQUISITION,
'document_validated': DialogPhase.VALIDATE,
'integration_status': DialogPhase.PROJECT_INTEGRATION,
'integration_action': DialogPhase.PROJECT_INTEGRATION,
'integration_error': DialogPhase.PROJECT_INTEGRATION,
'save_start': DialogPhase.SAVE,
'save_helper': DialogPhase.SAVE,
'save_progress': DialogPhase.SAVE,
'save_quit_recovered': DialogPhase.CLOSE,
'saved_and_closed': DialogPhase.CLOSE,
'pcb_exit_recovered': DialogPhase.CLOSE,
'complete': DialogPhase.CLOSE,
}
return mapping.get(stage, current or DialogPhase.PCB_STARTING)
def _diagnostic_close_event(job, event, target_pid):
close_id = event.get('close_id')
return (job.get('diagnostic_only') is True and event.get('stage') == 'diagnostic_close_requested'
and target_pid is not None and event.get('pcb_pid') == target_pid
and type(event.get('hwnd')) is int and event['hwnd'] > 0
and isinstance(close_id, str) and len(close_id) == 32 and all(c in '0123456789abcdef' for c in close_id)
and bool(job.get('operation_id')) and event.get('operation_id') == job['operation_id']
and all(job.get(key) and event.get(key) == job[key] for key in ('project', 'pcb', 'desktop')))
def _acknowledge_diagnostic_close(job, event, stop):
from .operation_queue import check_active
check_active()
if stop is not None and stop.is_set() or Path(job['cancel_signal']).exists():
raise InterruptedError('VERIFY_CANCELLED: diagnostic handoff cancelled')
ack = Path(job['root'])/'diagnostic-close-ack.json'
pending = ack.with_suffix('.pending')
pending.write_text(json.dumps({key: event[key] for key in
('pcb_pid', 'hwnd', 'operation_id', 'desktop', 'project', 'pcb', 'close_id')}), encoding='utf-8')
check_active()
if stop is not None and stop.is_set() or Path(job['cancel_signal']).exists():
raise InterruptedError('VERIFY_CANCELLED: diagnostic handoff cancelled')
pending.replace(ack)
def _close_intent_after_poll(job, events, offset, target_pid):
if not job.get('diagnostic_only') or not events.exists(): return None
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
for line in stream:
try: event = json.loads(line)
except json.JSONDecodeError: break
if _diagnostic_close_event(job, event, target_pid): return event
return None
def classify_verify_dispatch(*, verify_invoked, invoke_returned, target_pid, observation):
"""Classify a returned Verify command that never started its PCB.
UI Automation can report a successful ``InvokePattern`` call even when
Library Manager only closed the popup and did not dispatch its native MFC
command. A returned menu call is therefore not a success signal. Keep
this predicate small and evidence based so the supervisor can distinguish
a dispatch failure from a generic inactivity timeout without changing the
underlying automation route.
"""
if not (verify_invoked and invoke_returned) or target_pid is not None:
return None
return (
'FAIL',
'VERIFY_DISPATCH_FAILED: LMC Verify 菜单调用已返回,但未启动目标 ReusableCircuit PCB',
)
def _write_failure_windows(root, snapshots):
if not snapshots:
return
records = [dict(time=time.strftime('%Y-%m-%d %H:%M:%S'), **snapshot.to_record())
for snapshot in snapshots]
(root/'window_snapshots.jsonl').write_text(
''.join(json.dumps(record, ensure_ascii=False) + '\n' for record in records), encoding='utf-8')
def _write_process_tree(root, process, target_pid):
record = {
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
'worker_pid': getattr(process, 'pid', None),
'companion_pids': list(getattr(process, 'companion_pids', ())),
'pcb_pid': target_pid,
}
temporary = root/'process_tree.json.tmp'
temporary.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
temporary.replace(root/'process_tree.json')
def runtime_fingerprint(mentor_bin=''):
"""Return host/runtime facts needed to compare Win10 and Win11 runs."""
result = {
'platform': platform.platform(),
'system': platform.system(),
'release': platform.release(),
'version': platform.version(),
'machine': platform.machine(),
'python_version': platform.python_version(),
'python_bitness': struct.calcsize('P') * 8,
'mentor_bin': str(mentor_bin or ''),
'uia_backend': 'pywinauto.win32',
}
if os.name == 'nt':
try:
import winreg
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
r'SOFTWARE\Microsoft\Windows NT\CurrentVersion') as key:
for name, output_name in (
('ProductName', 'windows_product_name'),
('CurrentBuild', 'windows_current_build'),
('UBR', 'windows_ubr'),
):
try:
result[output_name] = winreg.QueryValueEx(key, name)[0]
except OSError:
pass
except (ImportError, OSError):
pass
result['windows_build_family'] = windows_build_family(
result.get('windows_current_build'))
from .mentor_paths import describe_installation, discover_sdd_home
result['mentor_installation'] = describe_installation(
discover_sdd_home(mentor_bin))
return result
def windows_build_family(build):
"""Classify Windows from the build number, not ProductName.
Windows 11 commonly keeps ``ProductName=Windows 10 Pro`` for
compatibility. The first public Windows 11 build is 22000, so the build
number is the stable discriminator for compatibility reports.
"""
try:
number = int(str(build or '').split('.', 1)[0])
except (TypeError, ValueError):
return 'unknown'
if number >= 22000:
return 'Windows 11'
if number >= 10240:
return 'Windows 10'
return 'unknown'
def native_lm_verify_enabled():
"""Read the explicit compatibility switch for the native LM route."""
return os.environ.get('CBB_NATIVE_LM_VERIFY', '').strip().casefold() in {
'1', 'true', 'yes', 'on',
}
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, launcher_pid=None,
progress=None):
"""Record native release or retention after automation has stopped."""
from .mentor_adapter import cleanup_verify_processes
event_path = Path(root) / 'events.jsonl'
def emit(**event):
event = {
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
'source': 'supervisor_cleanup',
**event,
}
try:
with event_path.open('a', encoding='utf-8') as stream:
stream.write(json.dumps(event, ensure_ascii=False) + '\n')
except OSError:
pass
if progress:
_notify_progress(progress, event)
return cleanup_verify_processes(
lm_pid,
target_pid,
launcher_pid,
expected_pcb=pcb,
baseline_pids=pcb_baseline,
emit=emit,
)
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None,
admission_timeout_seconds=10):
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
if stop is not None and stop.is_set() or context is not None and context.stop.is_set():
return VerifyResult('CANCELLED', stderr='已取消,未启动原生操作。', safe_to_continue=True)
options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner,
port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory,
version_prompt=version_prompt)
bound = current_context()
if bound is None:
# Management may call Verify while holding a same-thread lease. Do not
# transfer that transaction to a different thread.
import threading
from .library_lock import _guard, _owners
with _guard:
nested = any(v['thread'] == threading.get_ident() and v['depth'] > 0
for v in _owners.values())
if not nested:
from .operation_queue import request_library_keys
bound = context or BatchTaskContext(library=str(request.lmc_file),
aliases=tuple(sorted(request_library_keys(request))), targets=(request.block_name,))
try:
return bound.run(lambda: run_verify(request, stop=stop, **options),
stop=stop, timeout_seconds=admission_timeout_seconds)
except TaskHandoff as error:
result = _handoff_verify_result(error)
_notify_progress(bound.guard_callback(progress) if progress else None,
dict(stage='task_handoff', operation_id=bound.operation_id,
library=bound.library, status=result.status, resource_scope=error.scope))
return result
if bound is not None and progress is not None:
options['progress'] = bound.guard_callback(progress)
return _run_verify_task(request, stop=task_stop(stop), **options)
def _handoff_verify_result(error):
known = error.result if isinstance(error.result, VerifyResult) else None
result = known or VerifyResult('CANCELLED' if error.cancelled else 'TIMEOUT')
return replace(result, status='FAIL' if result.status == 'PASS' else result.status,
stderr=result.stderr or str(error), safe_to_continue=False,
error_stage=result.error_stage or 'admission_or_finalization',
error_category=result.error_category or 'TASK_HANDOFF',
release_proof=dict(lease_state='PENDING', resource_scope=error.scope,
blocked_resources=error.scope.get('resources', [])))
def _run_verify_task(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None, version_prompt=None, stop=None):
# Admission/finalization keep only the library lease. Native launch paths
# acquire the host channel after slow metadata has returned.
from .operation_queue import check_active
from .com_runtime import initialize_com, uninitialize_com
initialized = initialize_com()
library_lock = None
previous_release = None
release_confirmed = True
remaining_clients = ()
def clients_released():
if release_confirmed:
return True
from .library_lock import _supervisor_alive
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
try:
check_active()
if request.maintenance:
from .maintenance_entry import run_icdb_check
return run_icdb_check(request, progress=progress, stop=stop)
from .library_lock import LibraryLock, LibraryBusy, recheck_previous_release
import pywintypes
if Path(request.lmc_file).is_file():
try:
previous_release = recheck_previous_release(request.lmc_file,
Path(request.evidence_dir)/'previous-release', block_name=request.block_name,
progress=progress, stop=stop)
check_active()
library_lock = LibraryLock(request.lmc_file, checkpoint=False, block_name=request.block_name,
safe_to_release=clients_released)
library_lock.__enter__()
check_active()
except LibraryBusy as error:
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
except (OSError, ValueError, RuntimeError, pywintypes.error) as error:
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_ACCESS_FAILED')
release_confirmed = False
result = _run_verify_serialized(
request, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
facts = result.write_outcome or {}
if (not request.diagnostic_only and not request.auto_repair and request.post_repair_proof is None
and result.status == 'FAIL' and result.safe_to_continue and result.evidence_path
and result.library_flag == 'NOT_GREEN' and result.error_stage == 'library_flag'
and result.error_category == 'MENTOR_VERIFY_NOT_COMMITTED'
and all(facts.get(key) == value for key, value in (
('save_state','COMMITTED'), ('close_state','CLOSED'), ('helper_state','EXITED'),
('resource_state','RELEASED'), ('interrupted',False)))
and not (stop is not None and stop.is_set())):
# A new reusable design can save its Work data on the first open,
# then publish Cert on the next native Save. Never retry an open/write failure.
followup_path = result.evidence_path.with_name('commit-followup.json')
followup = dict(kind='NATIVE_COMMIT_FOLLOWUP', status='STARTED',
library=str(request.lmc_file), block=request.block_name, max_additional_runs=1,
original_evidence=str(result.evidence_path))
initial_seconds = result.timings_seconds.get('total', 0.0)
release_confirmed = True # Completed first run stays releasable until the next native attempt.
try:
followup_path.write_text(json.dumps(followup, ensure_ascii=False, indent=2), encoding='utf-8')
except OSError as error:
_notify_progress(progress, {'stage':'commit_followup_unavailable', 'block':request.block_name,
'detail':str(error)})
else:
check_active()
_notify_progress(progress, {'stage':'commit_followup', 'block':request.block_name,
'detail':'原生已保存关闭并释放,但 READY 尚未提交;补充一次原生校验。'})
check_active()
release_confirmed = False
result = _run_verify_serialized(request, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
followup.update(status='READY' if result.status == 'PASS' and result.library_flag == 'GREEN'
and result.safe_to_continue else 'UNCONFIRMED', final_evidence=str(result.evidence_path or ''))
_write_optional_evidence(followup_path, followup)
second_seconds = result.timings_seconds.get('total', 0.0)
result = replace(result, recovery_attempts=(*result.recovery_attempts, followup),
timings_seconds={**result.timings_seconds, 'initial_verify':initial_seconds,
'commit_followup':second_seconds, 'total':initial_seconds+second_seconds})
if result.evidence_path:
_write_optional_evidence(result.evidence_path, {'block_name':request.block_name, **asdict(result)})
if (request.auto_repair and result.status not in {'TIMEOUT', 'BLOCKED'}
and result.error_category == 'ICDB_REPAIR_REQUIRED'
and result.evidence_path):
from .native_repair import attempt_recovery
try:
diagnosis = json.loads((result.evidence_path.parent/'diagnostics/icdb-recovery.json').read_text('utf-8'))
original = result
snapshot = original.evidence_path.with_name('pre-repair-result.json')
snapshot.write_text(json.dumps(asdict(original), ensure_ascii=False, default=str, indent=2),
encoding='utf-8')
recovery = attempt_recovery(request, diagnosis, result.evidence_path.parent, progress=progress, stop=stop)
recovery = {**recovery, 'original_error': original.stderr,
'original_evidence': str(original.evidence_path),
'original_result_snapshot': str(snapshot),
'original_stage': original.error_stage,
'native_status': recovery.get('native_status', recovery.get('status')),
'validation_passed': False}
if recovery.get('status') == 'REPAIRED':
from .native_repair import native_success_confirmed, select_repair_target
try:
confirmed = native_success_confirmed(select_repair_target(request, diagnosis), recovery)
except (ValueError, OSError):
confirmed = False
if not confirmed:
release = recovery.get('release')
recovery.update(status='UNCONFIRMED',
release={**(release if isinstance(release, dict) else {}), 'safe_to_continue': False},
detail='NATIVE_REPAIR_EVIDENCE_UNCONFIRMED: 原生成功记录不完整或矛盾,未重试校验')
if (recovery.get('status') == 'REPAIRED'
and recovery.get('release', {}).get('safe_to_continue') is True):
result = replace(original, recovery_attempts=(recovery,))
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
result = _run_verify_serialized(
replace(request, auto_repair=False, maintenance=False), progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
recovery['validation_evidence'] = str(result.evidence_path or '')
recovery['validation_passed'] = (result.status == 'PASS'
and result.library_flag == 'GREEN' and result.safe_to_continue)
if not recovery['validation_passed']:
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
if result.status == 'PASS':
result = replace(result, status='FAIL',
stderr=(result.stderr+'\nREPAIR_VALIDATION_UNCONFIRMED: 未取得校验 GREEN').strip())
else:
recovery['status'] = 'REPAIRED'
else:
# A failed local database must not block unrelated CBBs once
# every owner is released. Re-check target ownership here so
# an incomplete repair record cannot authorize the queue.
safe = False
if (recovery.get('status') in {'MISSING_HEALTHY_BACKUP', 'NATIVE_REPAIR_FAILED'}
and recovery.get('evidence', {}).get('failure_complete') is True
and recovery.get('release', {}).get('safe_to_continue') is True):
from .native_repair import select_repair_target
try:
target = select_repair_target(request, diagnosis)
safe = (Path(recovery.get('project', '')).resolve() == target.project
and Path(recovery.get('database', '')).resolve() == target.database)
except (ValueError, OSError):
pass
result = replace(result, safe_to_continue=safe, stderr=(result.stderr + '\n'
+ str(recovery.get('status', 'REPAIR_UNCONFIRMED')) + ': '
+ str(recovery.get('detail', '原生 Repair 未完成恢复'))).strip())
result = replace(result, recovery_attempts=(recovery,))
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except Exception as error:
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status,
safe_to_continue=False,
stderr=(result.stderr+f'\nREPAIR_UNCONFIRMED: {error}').strip())
if result.evidence_path:
try:
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except OSError:
pass # Keep the native/repair errors even if evidence storage fails.
release_confirmed = result.safe_to_continue
if not release_confirmed and result.evidence_path:
try:
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
release_confirmed = (release.get('worker_exited') is True
and release.get('safe_to_continue') is True and release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve())
if (release.get('worker_exited') is True and not release.get('error')
and not release.get('close_errors') and not release.get('automation_release', {}).get('error')
and release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
remaining_clients = tuple(pid for pid in release.get('remaining_pids', [])
if type(pid) is int and pid > 0)
except (OSError, ValueError, KeyError):
pass
if previous_release is not None:
result = replace(result, release_proof=previous_release)
if result.evidence_path:
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
return result
finally:
try:
if library_lock is not None:
library_lock.__exit__(None, None, None)
finally:
uninitialize_com(initialized)
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
from .operation_queue import remember_result
timing = RunTimings()
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
if result.evidence_path:
try:
facts = json.loads(result.evidence_path.with_name('write-outcome.json').read_text('utf-8'))
session = result.evidence_path.with_name('session.json')
result = replace(result, write_outcome=facts,
session_record_path=session if session.is_file() else None)
except (OSError, ValueError):
pass
if not request.auto_repair and result.evidence_path:
try:
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
if (release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
released = release.get('worker_exited') is True and release.get('safe_to_continue') is True
result = replace(result, safe_to_continue=released)
if not released and result.status == 'PASS':
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
except (OSError, ValueError, KeyError):
pass
remember_result(result)
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
job_path = result.evidence_path.with_name('request.json')
if job_path.is_file():
timing.begin('failure_diagnostics')
try:
from .verify_diagnostics import capture_failure_logs_with_timeout
capture_failure_logs_with_timeout(job_path)
except Exception:
# Never replace the underlying Mentor/bridge/timeout failure.
pass
finally:
timing.end('failure_diagnostics')
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}:
from .icdb_recovery import ERROR_CODE, RECOVERY_ACTION, parse_recovery_request, recovery_detail
recovery = parse_recovery_request(result.stderr)
if recovery is None and result.evidence_path:
try:
saved = json.loads(result.evidence_path.with_name('diagnostics').joinpath(
'icdb-recovery.json').read_text('utf-8'))
recovery = parse_recovery_request(saved.get('native_message', ''))
except (OSError, ValueError, AttributeError):
pass
if recovery or ERROR_CODE.casefold() in result.stderr.casefold():
detail = result.stderr
if recovery and RECOVERY_ACTION not in detail:
detail = (detail + '\n' + recovery_detail(recovery)).strip()
result = replace(result, stderr=detail, error_category=ERROR_CODE)
if recovery and result.evidence_path:
try:
record_path = result.evidence_path.parent/'diagnostics/icdb-recovery.json'
record_path.parent.mkdir(parents=True, exist_ok=True)
if not record_path.is_file():
record_path.write_text(json.dumps(recovery, ensure_ascii=False, indent=2),
encoding='utf-8')
except OSError:
pass
durations, unfinished = timing.snapshot()
result = replace(result, timings_seconds=durations, incomplete_timing_phases=unfinished,
error_category=(result.error_category or classify_failure_category(result.stderr)
if result.status not in {"PASS", "SKIPPED"} else ""),
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
if result.status not in {"PASS", "SKIPPED"} else ""))
remember_result(result)
if result.evidence_path:
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
'block_name': request.block_name, 'status': result.status,
'unit': 'seconds', 'timings_seconds': durations,
'incomplete_timing_phases': unfinished,
'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)',
'notes': 'phase totals overlap action/call subphases; skipped phases are absent',
})
if progress:
_notify_progress(progress, {'stage': 'performance', 'block': request.block_name,
'timings_seconds': durations, 'incomplete_timing_phases': unfinished,
'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())})
if request.logging_context:
try:
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
except Exception:
logging.getLogger(__name__).warning('Operation logging failed', exc_info=True)
return result
def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None, version_prompt=None, stop=None):
"""Refresh one CBB and record its evidence.
``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``.
After successful execution, only IsVerifiedReuseBlock READY/raw=1 confirms
PASS/GREEN; Cert or icon observations never substitute for it. ``port_check`` is injectable so tests do
not depend on the host's port reservations.
"""
root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12])
try:
root.mkdir(parents=True, exist_ok=False)
except OSError as error:
return VerifyResult('FAIL', stderr=f'无法创建操作记录目录:{error}')
fingerprint = runtime_fingerprint(request.mentor_bin)
_write_optional_evidence(root/'runtime.json', fingerprint)
if progress:
_notify_progress(progress, {'stage': 'runtime_fingerprint', 'block': request.block_name,
**fingerprint})
result_path = root/'result.json'
runner = process_runner or (lambda command, timeout, cwd:
_run_process(command, timeout, cwd, evidence_root=root))
def finish(result):
# Only reclaim auxiliary backups after confirmed success. Failed runs
# retain every available original file for incident investigation.
try:
from .mentor_adapter import cleanup_evidence_backup
if result.status == 'PASS' and result.safe_to_continue:
cleanup_evidence_backup(root)
except Exception:
# Cleanup must never hide or change the verification result.
pass
from .operation_queue import remember_result
return remember_result(replace(result, evidence_path=result_path))
if stop is not None and stop.is_set():
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
timing.begin('preflight')
try:
project, pcb = preflight(request)
except Exception as error:
return finish(VerifyResult('BLOCKED', stderr=str(error)))
finally:
timing.end('preflight')
content_check_enabled = not request.diagnostic_only and bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
if content_check_enabled:
try:
executable = resolve_importer(request.mentor_bin)
except Exception as error:
return finish(VerifyResult('BLOCKED', stderr=str(error)))
command = build_content_check_command(request, executable)
if progress:
_notify_progress(progress, {'stage': 'content_check_start', 'block': request.block_name,
'detail': ' '.join(command)})
timing.begin('content_check')
try:
from .operation_queue import native_channel
with native_channel(read_only=True):
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
finally:
timing.end('content_check')
combined = '\n'.join(part for part in (stdout, stderr) if part)
report = contents_report_path(request.lmc_file, request.block_name)
try:
report.parent.mkdir(parents=True, exist_ok=True)
report.write_text(combined, encoding='utf-8')
except Exception as error:
stderr = f'{stderr}\n无法写入内容检查报告 {report}: {error}'
status = classify_output(exit_code, combined, timed_out=timed_out)
if status != 'PASS':
# A contents failure or an unusable DMS must block before touching the PCB.
return finish(VerifyResult(status if status in ('FAIL', 'TIMEOUT') else 'BLOCKED',
stderr=combined.strip() or f'内容检查结果未知(退出码 {exit_code})',
stdout=stdout, report_path=report, content_check_exit_code=exit_code,
safe_to_continue=not timed_out))
if progress:
_notify_progress(progress, {'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
# Sampling cannot establish that the whole native port range is unusable.
from .mentor_adapter import icdb_ports_bindable
check_ports = port_check or icdb_ports_bindable
ports_ok, ports_detail = check_ports()
if not ports_ok:
if progress:
_notify_progress(progress, {'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
if request.strict_port_check:
return finish(VerifyResult('BLOCKED', stderr=ports_detail))
job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name,
'diagnostic_only': request.diagnostic_only,
'supervisor_pid': os.getpid(),
'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds,
'mentor_bin': str(request.mentor_bin or ''),
'mapped_drive': str(request.mapped_drive or ''),
'mapped_remote_root': str(request.mapped_remote_root or ''),
'interactive_input': bool(request.interactive_input),
'native_lm_verify': bool(request.native_lm_verify),
'desktop': 'CbbVerify_' + uuid4().hex[:12],
'cancel_signal': str(root/'cancel-worker.signal'),
'start_gate': str(root/'start-worker.signal')}
job.update(operation_id=root.name, operation='verify', library=str(request.lmc_file),
block=request.block_name, session_record_path=str(root/'session.json'), start_gate_protocol='event-v1')
from .session_record import process_identity
job['supervisor_identity'] = process_identity(os.getpid())
consent = None
job_path = root/'request.json'
try:
if request.post_repair_proof is not None:
from .post_repair_consent import PostRepairProof, ConsentController, exact_prompt
if not isinstance(request.post_repair_proof, PostRepairProof):
return finish(VerifyResult('BLOCKED', stderr='POST_REPAIR_PROOF_REQUIRED'))
job['post_repair_consent'] = request.post_repair_proof.for_job(job)
consent = ConsentController(request.post_repair_proof, job)
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
except Exception as error:
return finish(VerifyResult('FAIL', stderr=str(error)))
events = root/'events.jsonl'
prewrite_eligible = not (request.diagnostic_only or request.interactive_input or request.native_lm_verify or consent)
def startup_phase(stage,**facts):
record={k:job[k] for k in ('operation_id','project','pcb','desktop')}
with events.open('a',encoding='utf-8') as stream:
stream.write(json.dumps(dict(stage=stage,**record,**facts))+'\n')
def before_dialog_action(snapshot,action):
name=action.get('action') if isinstance(action,dict) else action
if name not in ('dismiss_tip','accept_license','decline_database_log'):
startup_phase('startup_supervisor_write_possible',action=name,pid=snapshot.pid,hwnd=snapshot.hwnd)
offset = 0
target_pid = None
diagnostic_closing = False
launcher_pid = None
lm_pid = None
verify_invoked = False
lm_dispatch_deadline = None
invoke_returned = False
pcb_baseline = set()
last_observation = 0.0
last_post_verify_observation = {}
from .mentor_adapter import ActivityDeadline, pcb_startup_timeout_seconds
from .desktop_worker import DesktopWorker
from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend
from .dialog_models import DialogPhase
from .operation_queue import TaskHandoff, check_active, native_channel
check_active()
worker_launch_attempted = False
native = native_channel()
phase_entered = False
def end_native_phase():
nonlocal phase_entered
if phase_entered:
native.__exit__(None, None, None)
phase_entered = False
try:
if prewrite_eligible:startup_phase('startup_supervisor_no_write')
timing.begin('worker_setup')
with (root/'worker.log').open('w', encoding='utf-8') as log:
check_active()
worker_launch_attempted = True
if request.interactive_input:
process = DesktopWorker(
worker_command(job_path), None, interactive_input=True)
else:
process = DesktopWorker(worker_command(job_path), job['desktop'])
if hasattr(process, 'bind_cancellation'):
process.bind_cancellation(job['cancel_signal'])
if hasattr(process, 'bind_session'):
process.bind_session(job)
deadline = ActivityDeadline(request.timeout_seconds)
last_activity_key = None
phase = DialogPhase.PCB_STARTING
restore_controller = None
open_controller = None
dialog_agent = None
latest_snapshots = ()
if dialog_supervisor_factory is None:
if hasattr(process, 'start_companion'):
from .dialog_agent import DialogAgentClient
from .dialog_rules import default_rules_path, load_rules
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent', stop=stop)
dialog_supervisor = DialogSupervisor(
AgentDialogBackend(dialog_agent, before_action=before_dialog_action if prewrite_eligible else None, pids=lambda: {
int(pid) for pid in (target_pid, lm_pid, launcher_pid) if pid}),
rules=load_rules(default_rules_path(), diagnostic_only=request.diagnostic_only),
)
else:
dialog_supervisor = DialogSupervisor(Win32DialogBackend(
lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
desktop=process.desktop,before_action=before_dialog_action if prewrite_eligible else None,
))
else:
dialog_supervisor = dialog_supervisor_factory(
desktop=process.desktop,
pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
)
if stop is not None and stop.is_set():
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
check_active()
try:
native.__enter__()
phase_entered = True
except RuntimeError as error:
return finish(VerifyResult('BLOCKED', stderr=str(error), safe_to_continue=False,
error_category='NATIVE_CHANNEL_BUSY'))
from .operation_queue import commit_worker_start
commit_worker_start(process, job)
while process.poll() is None:
if stop is not None and stop.is_set():
save_markers = timing._save_markers
detail = ('保存已返回,关闭待确认' if 'SAVE_OK' in save_markers and 'CLOSE_OK' not in save_markers else
'已请求停止,等待当前保存返回' if phase == DialogPhase.SAVE else
'已请求停止,等待当前原生操作返回')
_notify_progress(progress, {'stage': 'save_stop_requested', 'block': request.block_name,
'detail': detail})
from .failed_session import cancelled_prewrite_startup
_stop_worker(process, wait_for_native_write=False if request.diagnostic_only else
lambda:cancelled_prewrite_startup(job,target_pid) is None)
return finish(VerifyResult('CANCELLED', stderr='已取消当前校验。',
safe_to_continue=False, error_stage=phase.value))
if events.exists():
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
while line := stream.readline():
try:
event = json.loads(line)
except json.JSONDecodeError:
break
timing.record(event)
if _diagnostic_close_event(job, event, target_pid):
_acknowledge_diagnostic_close(job, event, stop)
diagnostic_closing = True
if event.get('pcb_pid'): target_pid = event['pcb_pid']
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
if event.get('lm_pid'): lm_pid = event['lm_pid']
stage = event.get('stage')
if stage == 'lm_launch' and lm_dispatch_deadline is None:
from .mentor_adapter import lm_dispatch_timeout_seconds
lm_dispatch_deadline = (event.get('monotonic_seconds') or time.monotonic()) + lm_dispatch_timeout_seconds(request.timeout_seconds)
if stage == 'verify_invoke': verify_invoked = True
if stage == 'verify_invoke_returned': invoke_returned = True
if stage == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
phase = phase_for_event(event, phase)
activity_key = (stage, json.dumps(event.get('statuses'), sort_keys=True),
json.dumps(event.get('markers'), sort_keys=True),
event.get('action'), event.get('detail'))
# Repeated integration_status snapshots are normal
# polling, not progress. Only a changed state or
# a new stage/marker extends the inactivity window.
if activity_key != last_activity_key:
if stage in ('lm_started', 'lm_ready', 'verify_invoke',
'verify_invoke_returned', 'lm_session_ready',
'pcb_started', 'pcb_worker_started'):
deadline.timeout_seconds = pcb_startup_timeout_seconds(
request.timeout_seconds)
elif stage == 'identity_verified':
deadline.timeout_seconds = max(
0.1, float(request.timeout_seconds))
deadline.touch()
last_activity_key = activity_key
if progress: _notify_progress(progress, event)
offset = stream.tell()
if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5:
from .post_verify_diagnostics import observe_post_verify
last_observation = time.monotonic()
try:
observation = observe_post_verify(process.desktop, pcb, pcb_baseline, lm_pid)
except Exception as error:
observation = {'error': str(error)}
last_post_verify_observation = observation
if diagnostic_closing:
# The exact diagnostic worker owns its bounded close/No sequence.
from .dialog_supervisor import DialogPollResult
dialog_result = DialogPollResult(activity=False)
elif dialog_agent is not None and target_pid is None:
# The Library Manager Verify route starts its own window
# before it launches Expedition PCB. Do not feed that
# large MFC main window to the PCB dialog probe; it can
# exhaust the UIA observation deadline before Verify has
# even produced a target process.
from .dialog_supervisor import DialogPollResult
dialog_result = DialogPollResult(activity=False)
elif dialog_agent is not None:
dialog_result = dialog_supervisor.poll(phase)
elif dialog_supervisor_factory is not None:
try:
dialog_result = dialog_supervisor.poll(phase)
except TypeError:
# Compatibility for injected legacy test supervisors only.
dialog_result = dialog_supervisor.poll()
else:
dialog_result = dialog_supervisor.poll()
if not diagnostic_closing:
intent = _close_intent_after_poll(job, events, offset, target_pid)
if intent is not None:
_acknowledge_diagnostic_close(job, intent, stop)
diagnostic_closing = True
from .dialog_supervisor import DialogPollResult
dialog_result = DialogPollResult(activity=False)
if (version_prompt is not None and consent is None and dialog_agent is not None
and not request.diagnostic_only and not request.native_lm_verify
and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED'
and dialog_result.snapshot is not None):
from .pcb_open_consent import OpenController, exact_prompt
if exact_prompt(dialog_result.snapshot):
if open_controller is None:
open_controller = OpenController(job, auto_confirm=(
request.auto_confirm_pcb_open and not request.maintenance))
dialog_result = open_controller.handle(dialog_result.snapshot, dialog_result.observed_snapshots,
target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid,
show=version_prompt, perform=dialog_supervisor.backend.perform)
if (consent is not None and dialog_agent is not None
and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED'
and dialog_result.snapshot is not None and exact_prompt(dialog_result.snapshot)):
consent_snapshot = dialog_result.snapshot
if consent_snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}:
dialog_result = consent.handle(consent_snapshot, dialog_result.observed_snapshots,
target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid,
companion_pids=tuple(process.companion_pids), perform=dialog_supervisor.backend.perform)
if dialog_result.error_code == 'PCB_VERSION_CHOICE_REQUIRED' and dialog_result.snapshot is not None:
from .dialog_rules import archive_version_choice
choice_path = archive_version_choice(dialog_result.snapshot, root)
dialog_result = replace(dialog_result, detail=dialog_result.detail + f'\n版本候选与原生窗口证据:{choice_path}')
if (version_prompt is not None and not request.diagnostic_only and dialog_agent is not None
and dialog_result.snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}):
from .pcb_restore_prompt import RestoreController
if restore_controller is None:
restore_controller = RestoreController(job, user_open=consent is None)
dialog_result = restore_controller.handle(dialog_result.snapshot,
dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid,
worker_pid=process.pid, companion_pids=tuple(process.companion_pids),
show=version_prompt, perform=dialog_supervisor.backend.perform)
# Normal progress stays in memory. Only actions/errors and the
# last failure snapshots are persisted, not every elapsed tick.
latest_snapshots = dialog_result.observed_snapshots
if dialog_result.snapshot is not None and (
dialog_result.action_sent or dialog_result.error_code):
snapshot_record = (dialog_result.snapshot.to_record()
if hasattr(dialog_result.snapshot, 'to_record')
else dialog_result.snapshot.as_log_record())
record = {
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
**snapshot_record,
'action': dialog_result.action,
'rule_id': dialog_result.rule_id,
'channel': dialog_result.channel,
'accepted_control': list(dialog_result.accepted_controls),
'error_code': dialog_result.error_code,
'detail': dialog_result.detail,
}
with (root/'dialogs.jsonl').open('a', encoding='utf-8') as stream:
stream.write(json.dumps(record, ensure_ascii=False) + '\n')
if dialog_result.activity:
deadline.touch()
if dialog_result.error_code:
_write_failure_windows(root, latest_snapshots)
_write_process_tree(root, process, target_pid)
# A failed database load has no document to close. Give the
# worker time to normally close its owned empty PCB before
# recovery checks for remaining native database clients.
_stop_worker(process, timeout=40)
return finish(VerifyResult(
'CANCELLED' if dialog_result.error_code == 'USER_CANCELLED' else
'SKIPPED' if dialog_result.error_code == 'PCB_OPEN_SKIPPED' else 'FAIL',
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
error_category='PCB_OPEN_SKIPPED' if dialog_result.error_code == 'PCB_OPEN_SKIPPED' else '',
error_stage=phase.value,
safe_to_continue=False,
))
if deadline.expired() or (lm_dispatch_deadline is not None and target_pid is None
and time.monotonic() >= lm_dispatch_deadline):
_write_failure_windows(root, latest_snapshots)
# Stop automation without killing PCB/iCDB during a write.
_write_process_tree(root, process, target_pid)
if last_post_verify_observation:
(root/'post_verify_observations.jsonl').write_text(json.dumps({
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
'invoke_returned': invoke_returned,
**last_post_verify_observation,
}, ensure_ascii=False) + '\n', encoding='utf-8')
dispatch_failure = classify_verify_dispatch(
verify_invoked=verify_invoked,
invoke_returned=invoke_returned,
target_pid=target_pid,
observation=last_post_verify_observation,
)
_stop_worker(process, wait_for_native_write=not request.diagnostic_only)
if dispatch_failure is not None:
status, detail = dispatch_failure
if progress:
_notify_progress(progress, {
'stage': 'verify_dispatch_failed',
'block': request.block_name,
'detail': detail,
'observation': last_post_verify_observation,
})
return finish(VerifyResult(status, stderr=detail, safe_to_continue=False))
return finish(VerifyResult('TIMEOUT', stderr='操作超时,当前项未完成。', error_stage=phase.value, safe_to_continue=False))
time.sleep(.2)
end_native_phase() # Exact worker exit precedes all optional outcome/evidence I/O.
worker_result = root/'outcome.json'
if not worker_result.exists():
return finish(VerifyResult('FAIL', stderr=f'执行器异常退出 ({process.returncode}),见 worker.log', safe_to_continue=False))
outcome = json.loads(worker_result.read_text(encoding='utf-8'))
if outcome.get('status') == 'PASS' and process.returncode != 0:
return finish(VerifyResult('FAIL', stderr='执行器异常退出,不能确认成功', safe_to_continue=False))
result = VerifyResult(**outcome)
except Exception as error:
if 'process' in locals() and process.poll() is None:
_stop_worker(process, wait_for_native_write=not request.diagnostic_only)
launch_release = getattr(error, 'worker_launch_release', None) if 'process' not in locals() else None
if not worker_launch_attempted:
launch_release = {'process_created': False, 'worker_exited': True,
'safe_to_continue': True, 'remaining_pids': []}
no_launch = (isinstance(launch_release, dict)
and launch_release.get('process_created') is False
and launch_release.get('worker_exited') is True
and launch_release.get('safe_to_continue') is True
and launch_release.get('remaining_pids') == [])
if no_launch:
_write_optional_evidence(root/'session-release.json', {
**launch_release, 'block': request.block_name, 'library': str(request.lmc_file),
'worker_pid': None})
status = 'CANCELLED' if stop is not None and stop.is_set() else 'FAIL'
return finish(VerifyResult(status, stderr=str(error), safe_to_continue=no_launch))
finally:
close_errors = []
worker_exited = False
observer_started = False
automation_release = {'safe_to_continue': False}
if 'process' not in locals() or process.poll() is not None or getattr(process, 'session_record_path', None):
# A gated worker owns its writer barrier and inherited sidecar.
# Parent admission must not block controls needed for its exit.
end_native_phase()
if 'dialog_agent' in locals() and dialog_agent is not None:
try:
dialog_agent.close()
except Exception as error:
close_errors.append(str(error))
if 'process' in locals():
try:
worker_exited = process.poll() is not None
if not worker_exited:
worker_exited = _stop_worker(process, wait_for_native_write=not request.diagnostic_only)
if hasattr(process, 'automation_release_state'):
automation_release = process.automation_release_state()
else:
automation_release = {'safe_to_continue': worker_exited, 'remaining_pids': []}
if not worker_exited and getattr(process, 'session_record_path', None):
from .session_record import handoff_session
handoff_session(process.session_record_path, evidence_root=root,
process=process, progress=lambda state: _notify_progress(progress,
{'stage': 'session_observation', 'block': request.block_name, **state}))
observer_started = True
_notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name,
'detail': '已转后台观察,可查看状态'})
from .operation_queue import hold_native_handoff
hold_native_handoff(process, process.session_record_path)
except TaskHandoff as error:
handoff_result = _handoff_verify_result(error)
if handoff_result.evidence_path:
_write_optional_evidence(handoff_result.evidence_path,
{'block_name': request.block_name, **asdict(handoff_result)})
raise
except Exception as error:
close_errors.append(str(error))
if 'process' in locals():
try:
if events.is_file():
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
for line in stream:
try:
event = json.loads(line)
timing.record(event)
if event.get('pcb_pid'): target_pid = event['pcb_pid']
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
if event.get('lm_pid'): lm_pid = event['lm_pid']
if event.get('stage') == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
except (json.JSONDecodeError, ValueError):
continue
offset = stream.tell()
release_result = _cleanup_session_processes(
root,
lm_pid=lm_pid,
target_pid=target_pid,
launcher_pid=launcher_pid,
pcb=pcb,
pcb_baseline=pcb_baseline,
progress=progress,
)
except Exception as error:
release_result = {'safe_to_continue': False, 'error': str(error)}
try:
from .icdb_ownership import unreleased_addresses
addresses = unreleased_addresses(request.lmc_file, project, pcb)
if addresses:
release_result = {**(release_result or {}), 'icdb_addresses': addresses}
except OSError as error:
close_errors.append(str(error))
if close_errors:
release_result = {**(release_result or {}), 'safe_to_continue': False,
'close_errors': close_errors}
if not worker_exited or not automation_release.get('safe_to_continue'):
release_result = {**(release_result or {}), 'safe_to_continue': False,
'automation_release': automation_release, 'worker_exited': worker_exited}
release_result['remaining_pids'] = sorted(set(release_result.get('remaining_pids', []))
| set(automation_release.get('remaining_pids', [])))
if worker_exited:
try:
if (release_result.get('safe_to_continue') is not True
and getattr(process, 'session_record_path', None)):
if not observer_started:
from .session_record import handoff_session
# Retain the exact worker handle until registered
# native clients exit and the observer obtains proof.
handoff_session(process.session_record_path, evidence_root=root,
process=process, progress=lambda state: _notify_progress(progress,
{'stage': 'session_observation', 'block': request.block_name, **state}))
observer_started = True
_notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name,
'detail': '工作进程已退出,原生会话继续后台观察'})
else:
process.close()
except Exception as error:
release_result = {**release_result, 'safe_to_continue': False,
'close_errors': [*close_errors, str(error)]}
try:
from .operation_lifecycle import classify_save_state
save_lines = ((root/'save.log').read_text('utf-8', errors='replace').splitlines()
if (root/'save.log').is_file() else [])
_write_optional_evidence(root/'write-outcome.json', classify_save_state(save_lines,
helper_exited=(False if automation_release.get('remaining_pids') else
True if worker_exited and automation_release.get('safe_to_continue') else None),
native_released=release_result.get('safe_to_continue') is True))
(root/'session-release.json').write_text(json.dumps({
**(release_result or {}), 'block': request.block_name,
'library': str(request.lmc_file), 'worker_exited': worker_exited,
'worker_pid': getattr(process, 'pid', None)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
if worker_exited and release_result.get('safe_to_continue') is True and (root/'session.json').is_file():
from .session_record import write_session_record
write_session_record(root/'observation-supervisor.json', {
'operation_id': job['operation_id'], 'status': 'FINISHED',
'native_release_confirmed': True, 'worker_exited': True})
except OSError:
pass
if not release_result or not release_result.get('safe_to_continue'):
if 'result' in locals():
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False,
stderr=(result.stderr + '\nPCB_RELEASE_UNCONFIRMED: 未确认原生会话退出').strip())
recovery = {'desktop': job['desktop'], 'project': str(project),
'pcb': str(pcb), 'release': release_result,
'action': '停止批次;检查原生会话及数据库;不要删除锁文件或自动恢复。'}
try:
(root/'recovery-required.json').write_text(
json.dumps(recovery, ensure_ascii=False, indent=2), encoding='utf-8')
except OSError:
pass # Evidence failure must not prevent mutex release.
# Drain the timing tail even when the worker exited before a poll.
# This is optional telemetry: an unreadable log must not mask failure
# or prevent releasing the serial-operation mutex.
try:
if events.is_file():
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
for line in stream:
try:
timing.record(json.loads(line))
except (json.JSONDecodeError, ValueError):
continue
except OSError:
pass
finally:
end_native_phase()
if result.status == 'PASS' and not request.diagnostic_only:
timing.begin('library_flag')
bridge_records = []
def record_attempt(record):
bridge_records.append(record)
if progress:
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
try:
with native_channel(read_only=True):
probe = read_library_flag_with_retry(
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
request.block_name, on_attempt=record_attempt)
result = apply_post_verify_status(result, probe)
result = replace(result, bridge_attempts=tuple(bridge_records))
_write_optional_evidence(root/'bridge-status.json', bridge_records)
if progress:
_notify_progress(progress, {'stage': 'library_flag', 'block': request.block_name,
'library_flag': result.library_flag, 'detail': result.library_flag_detail})
finally:
timing.end('library_flag')
return finish(result)
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None,
context=None):
"""Run serially; isolate failures and skip writes to resources still retained."""
if max_workers != 1:
raise ValueError('Only serial Verify is supported for the shared Library Manager session')
if context is not None:
stop = stop if stop is not None else context.stop
progress = context.guard_callback(progress) if progress is not None else None
if verifier is None:
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available,
stop=stop, context=context) # noqa: E731
results = []
queue_stop = False
retained_libraries = {}
for request in requests:
from .operation_queue import request_library_keys
library_keys = request_library_keys(request, context)
for index in {retained_libraries[key][1] for key in library_keys.intersection(retained_libraries)}:
previous = next(value[0] for value in retained_libraries.values() if value[1] == index)
if queue_stop or stop is not None and stop.is_set():
break
if not results[index].evidence_path:
continue
from .operation_lifecycle import recheck_failed_release
refreshed = recheck_failed_release(previous.lmc_file, previous.block_name, results[index], stop=stop)
if refreshed.safe_to_continue:
results[index] = refreshed
retained_libraries = {k:v for k,v in retained_libraries.items() if v[1] != index}
if (queue_stop or stop is not None and stop.is_set()
or context is not None and context.stop.is_set()):
result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB')
elif library_keys.intersection(retained_libraries):
previous, index = retained_libraries[next(iter(library_keys.intersection(retained_libraries)))]
result = VerifyResult('SKIPPED', error_category='PREVIOUS_SESSION_UNRELEASED',
error_stage='admission', stderr=f'前项 {getattr(previous, "block_name", previous)} 的同库会话尚未释放,当前项未启动。')
else:
try:
def execute():
result = verifier(request)
if not result.safe_to_continue:
from .operation_lifecycle import recheck_failed_release
result = recheck_failed_release(request.lmc_file, request.block_name, result, stop=stop)
return result
result = item_executor(request, execute) if item_executor is not None else execute()
except Exception as error:
from .operation_queue import QueueCancelled, QueueUnavailable, TaskHandoff
# An unexpected exception cannot establish database safety.
result = (VerifyResult('SKIPPED', stderr=str(error), error_category='NATIVE_SESSION_PENDING',
error_stage='admission') if isinstance(error, QueueUnavailable) else
_handoff_verify_result(error) if isinstance(error, TaskHandoff) else
VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL',
stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled)))
results.append(result)
# Never start another writer after an unconfirmed native shutdown.
if not result.safe_to_continue:
retained_libraries.update({key:(request, len(results)-1) for key in library_keys})
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'}
if queue_stop and stop is not None:
stop.set()
if progress: _notify_progress(progress, request.block_name, result)
return results
def failure_report_path(batch_report):
batch_report = Path(batch_report)
return batch_report.with_name(batch_report.name.replace('batch-', 'failures-', 1))
def _report_field(value):
# One CBB per row; full native multi-line text remains in evidence files.
return str(value).replace('\r', r'\r').replace('\n', r'\n').replace('\t', r'\t')
def write_failure_report(requests, results, root, *, path=None):
requests, results = tuple(requests), tuple(results)
if len(requests) != len(results):
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
root = Path(root); root.mkdir(parents=True, exist_ok=True)
path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
failed = [(req, result) for req, result in zip(requests, results)
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}
and not (result.status in {'CHECKED', 'REPAIRED'} and result.safe_to_continue)]
passed = sum(result.status == 'PASS' for result in results)
checked = sum(result.status == 'CHECKED' and result.safe_to_continue for result in results)
repaired = sum(result.status == 'REPAIRED' and result.safe_to_continue for result in results)
skipped = len(results)-passed-len(failed)-checked-repaired
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped} 仅检查未修复: {checked} ICDB已修复待校验: {repaired}',
'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)']
if requests and all(req.maintenance for req in requests):
from .icdb_maintenance import maintenance_counts
counts = maintenance_counts(results)
lines[:2] = ['ICDB 检查异常清单',
f'总数: {len(results)} 未发现异常: {checked} 发现问题: {counts["issues_found"]} '
f'占用: {counts["in_use"]} 无法确认: {counts["unconfirmed"]} '
f'取消: {counts["cancelled"]} 未执行: {counts["skipped"]}']
for req, result in failed:
timings = ' '.join(f'{key}={value:.3f}s' +
('(unfinished)' if key in result.incomplete_timing_phases else '')
for key, value in result.timings_seconds.items())
lines.append(f'{req.block_name}\t{result.status}\t{result.error_category or classify_failure_category(result.stderr)}\t'
f'{result.error_stage or "unknown"}\t{result.library_flag}\t{_report_field(result.stderr or result.stdout.strip()[:200])}\t{result.evidence_path or ""}\t{timings}')
if not failed:
lines.append('本批次没有失败项。')
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
return path
def write_batch_report(requests, results, root):
requests, results = tuple(requests), tuple(results)
if len(requests) != len(results):
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
root = Path(root); root.mkdir(parents=True, exist_ok=True)
path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
lines = ['CBB Verify 批量结果',
'流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)']
if requests and all(req.maintenance for req in requests):
lines = ['ICDB 批量检查结果', '流程: 原生 PCB 打开检查;不执行 Repair、Restore 或 CBB 校验。']
for req, result in zip(requests, results):
detail = _report_field(result.stderr or result.stdout.strip()[:200])
seconds = ' '.join(f'{key}={value:.3f}s' +
('(unfinished)' if key in result.incomplete_timing_phases else '')
for key, value in result.timings_seconds.items())
lines.append(f'{req.block_name}\t{result.status}\t{result.library_flag}\t{detail}\t{result.evidence_path or ""}\t{seconds}')
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
write_failure_report(requests, results, root, path=failure_report_path(path))
return path