1340 lines
72 KiB
Python
1340 lines
72 KiB
Python
"""Serial Verify supervisor.
|
||
|
||
Each CBB gets one isolated, bounded automation worker on a private Windows
|
||
desktop. The worker reproduces the observed refresh flow: invoke the CBB's
|
||
native Library Manager Verify action, bind the resulting ReusableCircuit PCB,
|
||
verify the document identity, run Project Integration, save the Reusable Block
|
||
cell and close. Optionally the official
|
||
``ReuseBlockImporter.exe -chkContents`` check runs first when a LibSpec and a
|
||
DMS login configuration are supplied.
|
||
|
||
Nothing here injects mouse or keyboard input or edits the central library. The
|
||
native menu invocation happens on the worker's private desktop.
|
||
"""
|
||
from dataclasses import asdict, dataclass, field, replace
|
||
from pathlib import Path
|
||
import json
|
||
import logging
|
||
import os
|
||
import platform
|
||
import re
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from uuid import uuid4
|
||
|
||
from .network_path import absolute_preserving_drive
|
||
from .runtime_timing import RunTimings
|
||
from .defaults import DEFAULT_TIMEOUT_SECONDS
|
||
|
||
SUCCESS_TEXT = re.compile(r'content checking completed successfully', re.IGNORECASE)
|
||
ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE)
|
||
INVALID_NAME_CHARS = '\\/:*?"<>|'
|
||
|
||
|
||
def _notify_progress(callback, *args):
|
||
"""A failed observer cannot change native execution or release facts."""
|
||
if callback is not None:
|
||
try:
|
||
callback(*args)
|
||
except Exception:
|
||
logging.getLogger(__name__).warning('Progress notification failed', exc_info=True)
|
||
|
||
|
||
def _write_optional_evidence(path, value):
|
||
try:
|
||
path.write_text(json.dumps(value, ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||
except OSError:
|
||
logging.getLogger(__name__).warning('Could not write diagnostic evidence %s', path, exc_info=True)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VerifyRequest:
|
||
lmc_file: Path
|
||
block_name: str
|
||
evidence_dir: Path
|
||
libspec: str = ''
|
||
dms_login_config: str = ''
|
||
mentor_bin: str = ''
|
||
# PCB startup on EE7.9 includes iCDB recovery and COM registration. The
|
||
# previous 30-second UI default was shorter than the observed startup
|
||
# window on this machine and caused a false timeout before document attach.
|
||
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
|
||
logging_context: object | None = None
|
||
# The production default stays on an isolated desktop. The explicit
|
||
# interactive mode is reserved for a real-input compatibility test on
|
||
# machines where EE7.9 cannot dispatch the tree menu on a private desktop.
|
||
interactive_input: bool = False
|
||
# The native Library Manager menu route is opt-in until a machine's menu
|
||
# dispatch has been proven. The flag is serialized to the worker so a
|
||
# Win10 compatibility run can select it explicitly.
|
||
native_lm_verify: bool = False
|
||
# A reserved iCDB range is a host-wide prerequisite failure. Do not
|
||
# launch a PCB that is guaranteed to time out; batch orchestration will
|
||
# record this item and continue with the next one.
|
||
strict_port_check: bool = False
|
||
# A mapped drive can be invisible to an elevated/private worker on Win10.
|
||
# Preserve the parent session's mapping identity so the worker can restore
|
||
# it before launching Library Manager/PCB.
|
||
mapped_drive: str = ''
|
||
mapped_remote_root: str = ''
|
||
auto_repair: bool = False
|
||
maintenance: bool = False
|
||
post_repair_proof: object | None = field(default=None, repr=False, compare=False)
|
||
diagnostic_only: bool = False
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VerifyResult:
|
||
status: str
|
||
stderr: str = ''
|
||
stdout: str = ''
|
||
evidence_path: Path | None = None
|
||
safe_to_continue: bool = True
|
||
library_flag: str = 'UNKNOWN'
|
||
library_flag_detail: str = ''
|
||
report_path: Path | None = None
|
||
content_check_exit_code: int | None = None
|
||
error_category: str = ""
|
||
error_stage: str = ""
|
||
bridge_attempts: tuple[dict, ...] = ()
|
||
timings_seconds: dict[str, float] = field(default_factory=dict)
|
||
incomplete_timing_phases: tuple[str, ...] = ()
|
||
recovery_attempts: tuple[dict, ...] = ()
|
||
release_proof: dict | None = None
|
||
|
||
|
||
def resolve_importer(mentor_bin: str | None = None) -> Path:
|
||
"""Configured bin first, then the EE7.9.5 environment defaults."""
|
||
candidates = []
|
||
if mentor_bin:
|
||
configured = Path(mentor_bin)
|
||
if configured.is_file() and configured.name.casefold() == 'reuseblockimporter.exe':
|
||
candidates.append(configured.parent)
|
||
else:
|
||
candidates.append(configured)
|
||
if configured.name.casefold() == 'sdd_home':
|
||
candidates.append(configured / 'lm' / 'win32' / 'bin')
|
||
from .mentor_paths import discover_sdd_home
|
||
discovered = discover_sdd_home(mentor_bin)
|
||
if discovered is not None:
|
||
candidates.append(discovered / 'lm' / 'win32' / 'bin')
|
||
for variable in ('SDD_LM_BIN', 'SDD_HOME'):
|
||
value = os.environ.get(variable)
|
||
if not value:
|
||
continue
|
||
root = Path(value)
|
||
candidates.append(root if root.name.lower() == 'bin' else root/'lm'/'win32'/'bin')
|
||
candidates.append(Path(r'D:\Mentor\7.9.5EE\SDD_HOME\lm\win32\bin'))
|
||
for candidate in candidates:
|
||
executable = candidate/'ReuseBlockImporter.exe'
|
||
if executable.is_file():
|
||
return executable
|
||
raise ValueError('ReuseBlockImporter.exe 未找到;请设置 SDD_LM_BIN')
|
||
|
||
|
||
def build_content_check_command(request, executable) -> tuple[str, ...]:
|
||
"""The documented Logical-Physical contents check invocation."""
|
||
return (str(executable), '-lmcFile', str(request.lmc_file), '-blockName', request.block_name,
|
||
'-logicalType', 'LP', '-chkContents', '-libSpec', str(request.libspec),
|
||
'-dmsLoginConfig', str(request.dms_login_config))
|
||
|
||
|
||
def contents_report_path(lmc_file, block_name) -> Path:
|
||
safe_name = re.sub(f'[{re.escape(INVALID_NAME_CHARS)}]', '_', str(block_name))
|
||
return Path(lmc_file).parent/'LogFiles'/f'restorerb_RULP_{safe_name}_contents.log'
|
||
|
||
|
||
def classify_output(exit_code: int | None, output: str, *, timed_out: bool = False) -> str:
|
||
"""Never invent success: PASS needs the success text and no error marker."""
|
||
if timed_out:
|
||
return 'TIMEOUT'
|
||
from .icdb_recovery import parse_session_loss
|
||
if parse_session_loss(output):
|
||
return 'FAIL'
|
||
diagnostic_text = re.sub(r'\b0\s+errors?\b', '', output, flags=re.I)
|
||
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(diagnostic_text):
|
||
return 'PASS'
|
||
if ERROR_TEXT.search(output):
|
||
return 'FAIL'
|
||
if exit_code not in (None, 0):
|
||
return 'FAIL'
|
||
return 'UNKNOWN'
|
||
|
||
|
||
def preflight(request):
|
||
"""Validate the library entry and its ReusableCircuit sources before launch."""
|
||
library = absolute_preserving_drive(request.lmc_file)
|
||
name = request.block_name
|
||
if not name or name in ('.', '..') or any(character in name for character in INVALID_NAME_CHARS):
|
||
raise ValueError('CBB 名称非法')
|
||
if not library.is_file():
|
||
raise ValueError(f'中心库不存在: {library}')
|
||
work = library.parent/'ReusableBlocksLP'/name/'Work'
|
||
projects = list(work.glob('*.prj'))
|
||
if len(projects) != 1:
|
||
raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ')
|
||
project = absolute_preserving_drive(projects[0])
|
||
from .backup_recovery import _project_text
|
||
content = _project_text(project)
|
||
refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M)
|
||
if not refs:
|
||
raise ValueError(f'{name}: 工程没有 PCBDesignPath')
|
||
pcb = absolute_preserving_drive(project.parent/refs[0])
|
||
if not pcb.is_relative_to(absolute_preserving_drive(work)):
|
||
raise ValueError(f'{name}: PCB 引用越出当前 CBB,停止操作: {pcb}')
|
||
real_work = work.resolve()
|
||
expected_work = library.parent.resolve() / 'ReusableBlocksLP' / name / 'Work'
|
||
if os.path.normcase(str(real_work)) != os.path.normcase(str(expected_work)):
|
||
raise ValueError(f'{name}: Work 路径通过链接指向其他工程,停止操作')
|
||
for candidate in (project, pcb, pcb.parent/'pcb.prj', pcb.parent/'Layout',
|
||
pcb.parent/'Layout'/'LayoutDB.lyt', work/'database', work/'CES'):
|
||
if not candidate.resolve().is_relative_to(real_work):
|
||
raise ValueError(f'{name}: 文件或目录链接越出当前 CBB,停止操作: {candidate}')
|
||
if not pcb.is_file() or not (pcb.parent/'Layout'/'LayoutDB.lyt').is_file():
|
||
raise ValueError(f'{name}: 缺少 PCB 或布局数据库: {pcb}')
|
||
return project, pcb
|
||
|
||
|
||
def preflight_skip_reason(lmc_file, block_name):
|
||
"""Return a user-facing skip reason for an import-incomplete CBB.
|
||
|
||
This is intentionally narrower than ``preflight``: only local source
|
||
structure problems are excluded before a batch starts. Runtime failures
|
||
such as ports, DMS, COM, or Project Integration remain normal per-item
|
||
failures and are still attempted when the user queues them.
|
||
"""
|
||
library = absolute_preserving_drive(lmc_file)
|
||
if not library.is_file():
|
||
return ''
|
||
work = library.parent / 'ReusableBlocksLP' / str(block_name) / 'Work'
|
||
if not work.is_dir():
|
||
return f'{block_name}: 缺少 Work 工程目录'
|
||
request = VerifyRequest(library, str(block_name), library.parent / '.cbb-preflight-evidence')
|
||
try:
|
||
preflight(request)
|
||
except Exception as error:
|
||
detail = str(error)
|
||
if any(marker in detail for marker in ('缺少或无法唯一确定 Work 下的工程 PRJ',
|
||
'工程没有 PCBDesignPath',
|
||
'缺少 PCB 或布局数据库')):
|
||
return detail
|
||
return ''
|
||
|
||
|
||
def _run_process(command, timeout_seconds, cwd, *, evidence_root=None):
|
||
from .content_check import run_content_check
|
||
return run_content_check(command, timeout_seconds, cwd, evidence_root=evidence_root)
|
||
|
||
|
||
def _stop_worker(process, *, timeout=5):
|
||
"""Stop automation clients; native database owners are preserved."""
|
||
try:
|
||
if hasattr(process, 'request_stop'):
|
||
process.request_stop()
|
||
else:
|
||
process.kill()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
process.wait(timeout=timeout)
|
||
except Exception:
|
||
try:
|
||
process.kill() # Automation clients only; never the native tree.
|
||
process.wait(timeout=5)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if hasattr(process, 'stop_automation_clients'):
|
||
process.stop_automation_clients()
|
||
return process.poll() is not None
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _normalize_bridge_result(probe):
|
||
from .status_bridge import BridgeResult
|
||
if not isinstance(probe, BridgeResult):
|
||
detail = str(probe[1]) if isinstance(probe, tuple) and len(probe) > 1 else str(probe)
|
||
return BridgeResult('MALFORMED', '', None,
|
||
'BRIDGE_RESULT_INVALID: 非权威桥接器结果;' + detail)
|
||
if probe.status in {'READY', 'PENDING'} and (
|
||
probe.source != 'IsVerifiedReuseBlock'
|
||
or probe.raw_status != (1 if probe.status == 'READY' else 0)):
|
||
return BridgeResult('MALFORMED', probe.source, probe.raw_status,
|
||
'BRIDGE_RESULT_INVALID: status/source/raw_status 不一致')
|
||
return probe
|
||
|
||
|
||
def read_library_flag_with_retry(reader, library, block_name, *, attempts=3,
|
||
delays=(0.0, 3.0, 5.0), sleep=None, on_attempt=None):
|
||
"""Bounded read-only bridge retries; never repeat Verify or promote PENDING."""
|
||
from .status_bridge import BridgeResult
|
||
sleep = sleep or time.sleep
|
||
last = BridgeResult('UNAVAILABLE', '', None, '状态桥没有返回结果')
|
||
for index in range(max(1, int(attempts))):
|
||
if index > 0:
|
||
delay = delays[min(index, len(delays)-1)] if delays else 0
|
||
sleep(max(0.0, float(delay)))
|
||
started = time.monotonic()
|
||
try:
|
||
last = _normalize_bridge_result(reader(library, block_name))
|
||
except Exception as error:
|
||
last = BridgeResult('UNAVAILABLE', '', None, 'BRIDGE_UNAVAILABLE: ' + str(error))
|
||
if on_attempt:
|
||
on_attempt({'attempt': index + 1, 'status': last.status,
|
||
'source': last.source, 'raw_status': last.raw_status,
|
||
'detail': last.detail,
|
||
'query_seconds': round(time.monotonic()-started, 6)})
|
||
if last.status in {'READY', 'MALFORMED'}:
|
||
return last
|
||
return last
|
||
|
||
|
||
def classify_failure_category(detail):
|
||
text = str(detail or '')
|
||
lower = text.casefold()
|
||
if 'pcb_version_choice_required' in lower:
|
||
return 'PCB_VERSION_CHOICE_REQUIRED'
|
||
from .icdb_recovery import ERROR_CODE, parse_session_loss
|
||
if ERROR_CODE.casefold() in lower or parse_session_loss(text):
|
||
return ERROR_CODE
|
||
if 'back annotation is not allowed on verified (or once verified) reusable blocks.' in lower:
|
||
# Preserve the native integration failure even when shutdown adds a
|
||
# secondary release warning. This category does not grant permission
|
||
# to reimport or change safe_to_continue.
|
||
return 'CBB_REIMPORT_REQUIRED'
|
||
if any(code in lower for code in (
|
||
'pcb_release_unconfirmed', 'icdb_release_unconfirmed',
|
||
'icdb_session_unconfirmed', 'pcb_session_active', 'database_recovery_required')):
|
||
return 'DATABASE_SAFETY_STOP'
|
||
if 'tf_floatinglangbar_wndtitle' in lower or 'cicerouiwndframe' in lower:
|
||
return 'TOOL_FALSE_DIALOG'
|
||
if 'could not load job preferences db' in lower or 'unable to create local cell library' in lower:
|
||
return 'MENTOR_LOCAL_CELL_LIBRARY'
|
||
if ('database_load_failed' in lower
|
||
or 'problem encountered while opening one of the database files' in lower
|
||
or 'database load is being terminated' in lower):
|
||
return 'MENTOR_DATABASE_LOAD_FAILED'
|
||
if 'project integration error' in lower or 'project_integration_failed' in lower:
|
||
return 'MENTOR_PROJECT_INTEGRATION'
|
||
if 'save_reusable_block_failed' in lower or 'error saving the reusable block' in lower:
|
||
return 'MENTOR_SAVE_FAILED'
|
||
if 'design_server_failed' in lower or 'could not run server' in lower or 'could not bring it alive' in lower:
|
||
return 'ENVIRONMENT_DESIGN_SERVER'
|
||
if 'bridge_unavailable' in lower or '权威状态读取失败' in text:
|
||
return 'BRIDGE_UNAVAILABLE'
|
||
if 'bridge_result_invalid' in lower:
|
||
return 'BRIDGE_RESULT_INVALID'
|
||
if 'pending' in lower and 'isverifiedreuseblock' in lower:
|
||
return 'MENTOR_VERIFY_NOT_COMMITTED'
|
||
if 'cert' in lower and ('incomplete' in lower or '成果' in text):
|
||
return 'TOOL_CERT_DIAGNOSTIC'
|
||
if 'unknown_dialog' in lower or 'dialog' in lower:
|
||
return 'TOOL_DIALOG_SUPERVISOR'
|
||
if 'timeout' in lower or '超时' in text:
|
||
return 'TOOL_TIMEOUT'
|
||
return 'UNCLASSIFIED'
|
||
|
||
|
||
def read_library_flag_if_available(library, block_name):
|
||
"""Read the authoritative Mentor Verify state through the x86 bridge.
|
||
|
||
Unit tests replace this external boundary; production never accepts Cert
|
||
files or legacy tree readers as a substitute for this gold standard.
|
||
"""
|
||
from .status_bridge import StatusBridgeClient
|
||
return StatusBridgeClient(
|
||
env={'CBB_STATUS_BRIDGE_ALLOW_PRIVATE': '1'}
|
||
).query(Path(library), block_name)
|
||
|
||
|
||
def apply_post_verify_status(result, bridge_result):
|
||
"""Only the authoritative bridge can confirm READY, never a Cert/tuple/icon."""
|
||
if result.status != 'PASS':
|
||
return result # a pre-existing READY cannot erase a real execution error
|
||
bridge_result = _normalize_bridge_result(bridge_result)
|
||
if bridge_result.status == 'READY':
|
||
return replace(result, library_flag='GREEN', library_flag_detail=(
|
||
f'READY ({bridge_result.source}, raw={bridge_result.raw_status})'))
|
||
detail = bridge_result.detail or (
|
||
f'执行已完成,但权威状态仍为 {bridge_result.status} '
|
||
f'({bridge_result.source}, raw={bridge_result.raw_status})')
|
||
pending = bridge_result.status == 'PENDING'
|
||
return replace(result, status='FAIL',
|
||
stderr='\n'.join(part for part in (result.stderr, detail) if part),
|
||
library_flag='NOT_GREEN' if pending else 'UNKNOWN', library_flag_detail=detail,
|
||
error_category=('MENTOR_VERIFY_NOT_COMMITTED' if pending else
|
||
'BRIDGE_RESULT_INVALID' if bridge_result.status == 'MALFORMED' else 'BRIDGE_UNAVAILABLE'),
|
||
error_stage='library_flag', safe_to_continue=True)
|
||
|
||
|
||
def worker_command(request_path):
|
||
if getattr(sys, 'frozen', False):
|
||
return [sys.executable, '--verify-worker', str(request_path)]
|
||
return [sys.executable, '-m', 'cbb_refresh_verify', '--verify-worker', str(request_path)]
|
||
|
||
|
||
def phase_for_event(event, current=None):
|
||
from .dialog_models import DialogPhase
|
||
|
||
stage = str(event.get('stage', ''))
|
||
mapping = {
|
||
'lm_launch': DialogPhase.PCB_STARTING,
|
||
'lm_started': DialogPhase.PCB_STARTING,
|
||
'lm_ready': DialogPhase.PCB_STARTING,
|
||
'lm_session_ready': DialogPhase.PCB_STARTING,
|
||
'verify_invoke': DialogPhase.PCB_STARTING,
|
||
'verify_invoke_returned': DialogPhase.PCB_STARTING,
|
||
'pcb_launch': DialogPhase.PCB_STARTING,
|
||
'pcb_started': DialogPhase.PCB_STARTING,
|
||
'com_wait': DialogPhase.DOCUMENT_ATTACH,
|
||
'identity_wait': DialogPhase.DOCUMENT_ATTACH,
|
||
'identity_verified': DialogPhase.IDENTITY_CHECK,
|
||
'license_key_requested': DialogPhase.LICENSE_ACQUISITION,
|
||
'license_token_received': DialogPhase.LICENSE_ACQUISITION,
|
||
'document_validated': DialogPhase.VALIDATE,
|
||
'integration_status': DialogPhase.PROJECT_INTEGRATION,
|
||
'integration_action': DialogPhase.PROJECT_INTEGRATION,
|
||
'integration_error': DialogPhase.PROJECT_INTEGRATION,
|
||
'save_start': DialogPhase.SAVE,
|
||
'save_helper': DialogPhase.SAVE,
|
||
'save_progress': DialogPhase.SAVE,
|
||
'save_quit_recovered': DialogPhase.CLOSE,
|
||
'saved_and_closed': DialogPhase.CLOSE,
|
||
'pcb_exit_recovered': DialogPhase.CLOSE,
|
||
'complete': DialogPhase.CLOSE,
|
||
}
|
||
return mapping.get(stage, current or DialogPhase.PCB_STARTING)
|
||
|
||
|
||
def classify_verify_dispatch(*, verify_invoked, invoke_returned, target_pid, observation):
|
||
"""Classify a returned Verify command that never started its PCB.
|
||
|
||
UI Automation can report a successful ``InvokePattern`` call even when
|
||
Library Manager only closed the popup and did not dispatch its native MFC
|
||
command. A returned menu call is therefore not a success signal. Keep
|
||
this predicate small and evidence based so the supervisor can distinguish
|
||
a dispatch failure from a generic inactivity timeout without changing the
|
||
underlying automation route.
|
||
"""
|
||
if not (verify_invoked and invoke_returned) or target_pid is not None:
|
||
return None
|
||
return (
|
||
'FAIL',
|
||
'VERIFY_DISPATCH_FAILED: LMC Verify 菜单调用已返回,但未启动目标 ReusableCircuit PCB',
|
||
)
|
||
|
||
|
||
def _write_failure_windows(root, snapshots):
|
||
if not snapshots:
|
||
return
|
||
records = [dict(time=time.strftime('%Y-%m-%d %H:%M:%S'), **snapshot.to_record())
|
||
for snapshot in snapshots]
|
||
(root/'window_snapshots.jsonl').write_text(
|
||
''.join(json.dumps(record, ensure_ascii=False) + '\n' for record in records), encoding='utf-8')
|
||
|
||
|
||
def _write_process_tree(root, process, target_pid):
|
||
record = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'worker_pid': getattr(process, 'pid', None),
|
||
'companion_pids': list(getattr(process, 'companion_pids', ())),
|
||
'pcb_pid': target_pid,
|
||
}
|
||
temporary = root/'process_tree.json.tmp'
|
||
temporary.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
temporary.replace(root/'process_tree.json')
|
||
|
||
|
||
def runtime_fingerprint(mentor_bin=''):
|
||
"""Return host/runtime facts needed to compare Win10 and Win11 runs."""
|
||
result = {
|
||
'platform': platform.platform(),
|
||
'system': platform.system(),
|
||
'release': platform.release(),
|
||
'version': platform.version(),
|
||
'machine': platform.machine(),
|
||
'python_version': platform.python_version(),
|
||
'python_bitness': struct.calcsize('P') * 8,
|
||
'mentor_bin': str(mentor_bin or ''),
|
||
'uia_backend': 'pywinauto.win32',
|
||
}
|
||
if os.name == 'nt':
|
||
try:
|
||
import winreg
|
||
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
|
||
r'SOFTWARE\Microsoft\Windows NT\CurrentVersion') as key:
|
||
for name, output_name in (
|
||
('ProductName', 'windows_product_name'),
|
||
('CurrentBuild', 'windows_current_build'),
|
||
('UBR', 'windows_ubr'),
|
||
):
|
||
try:
|
||
result[output_name] = winreg.QueryValueEx(key, name)[0]
|
||
except OSError:
|
||
pass
|
||
except (ImportError, OSError):
|
||
pass
|
||
result['windows_build_family'] = windows_build_family(
|
||
result.get('windows_current_build'))
|
||
from .mentor_paths import describe_installation, discover_sdd_home
|
||
result['mentor_installation'] = describe_installation(
|
||
discover_sdd_home(mentor_bin))
|
||
return result
|
||
|
||
|
||
def windows_build_family(build):
|
||
"""Classify Windows from the build number, not ProductName.
|
||
|
||
Windows 11 commonly keeps ``ProductName=Windows 10 Pro`` for
|
||
compatibility. The first public Windows 11 build is 22000, so the build
|
||
number is the stable discriminator for compatibility reports.
|
||
"""
|
||
try:
|
||
number = int(str(build or '').split('.', 1)[0])
|
||
except (TypeError, ValueError):
|
||
return 'unknown'
|
||
if number >= 22000:
|
||
return 'Windows 11'
|
||
if number >= 10240:
|
||
return 'Windows 10'
|
||
return 'unknown'
|
||
|
||
|
||
def native_lm_verify_enabled():
|
||
"""Read the explicit compatibility switch for the native LM route."""
|
||
return os.environ.get('CBB_NATIVE_LM_VERIFY', '').strip().casefold() in {
|
||
'1', 'true', 'yes', 'on',
|
||
}
|
||
|
||
|
||
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, launcher_pid=None,
|
||
progress=None):
|
||
"""Record native release or retention after automation has stopped."""
|
||
from .mentor_adapter import cleanup_verify_processes
|
||
|
||
event_path = Path(root) / 'events.jsonl'
|
||
|
||
def emit(**event):
|
||
event = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'source': 'supervisor_cleanup',
|
||
**event,
|
||
}
|
||
try:
|
||
with event_path.open('a', encoding='utf-8') as stream:
|
||
stream.write(json.dumps(event, ensure_ascii=False) + '\n')
|
||
except OSError:
|
||
pass
|
||
if progress:
|
||
_notify_progress(progress, event)
|
||
|
||
return cleanup_verify_processes(
|
||
lm_pid,
|
||
target_pid,
|
||
launcher_pid,
|
||
expected_pcb=pcb,
|
||
baseline_pids=pcb_baseline,
|
||
emit=emit,
|
||
)
|
||
|
||
|
||
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||
# Cover contents checking, native work and final state reads with one
|
||
# session-wide mutex. The worker-stage lock below is reentrant on Windows.
|
||
import win32event, win32con
|
||
from .com_runtime import initialize_com, uninitialize_com
|
||
initialized = initialize_com()
|
||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||
acquired = False
|
||
library_lock = None
|
||
previous_release = None
|
||
release_confirmed = True
|
||
remaining_clients = ()
|
||
def clients_released():
|
||
if release_confirmed:
|
||
return True
|
||
from .library_lock import _supervisor_alive
|
||
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
|
||
try:
|
||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||
if not acquired:
|
||
return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。',
|
||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')
|
||
if request.maintenance:
|
||
from .maintenance_entry import run_icdb_check
|
||
return run_icdb_check(request, progress=progress, stop=stop)
|
||
from .library_lock import LibraryLock, LibraryBusy, recheck_previous_release
|
||
import pywintypes
|
||
if Path(request.lmc_file).is_file():
|
||
try:
|
||
previous_release = recheck_previous_release(request.lmc_file,
|
||
Path(request.evidence_dir)/'previous-release', block_name=request.block_name, progress=progress)
|
||
library_lock = LibraryLock(request.lmc_file, checkpoint=False,
|
||
safe_to_release=clients_released)
|
||
library_lock.__enter__()
|
||
except LibraryBusy as error:
|
||
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
|
||
except (OSError, ValueError, RuntimeError, pywintypes.error) as error:
|
||
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_ACCESS_FAILED')
|
||
release_confirmed = False
|
||
result = _run_verify_serialized(
|
||
request, progress=progress, flag_reader=flag_reader,
|
||
process_runner=process_runner, port_check=port_check,
|
||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
|
||
if (request.auto_repair and result.status not in {'TIMEOUT', 'BLOCKED'}
|
||
and result.error_category == 'ICDB_REPAIR_REQUIRED'
|
||
and result.evidence_path):
|
||
from .native_repair import attempt_recovery
|
||
try:
|
||
diagnosis = json.loads((result.evidence_path.parent/'diagnostics/icdb-recovery.json').read_text('utf-8'))
|
||
original = result
|
||
snapshot = original.evidence_path.with_name('pre-repair-result.json')
|
||
snapshot.write_text(json.dumps(asdict(original), ensure_ascii=False, default=str, indent=2),
|
||
encoding='utf-8')
|
||
recovery = attempt_recovery(request, diagnosis, result.evidence_path.parent, progress=progress)
|
||
recovery = {**recovery, 'original_error': original.stderr,
|
||
'original_evidence': str(original.evidence_path),
|
||
'original_result_snapshot': str(snapshot),
|
||
'original_stage': original.error_stage,
|
||
'native_status': recovery.get('native_status', recovery.get('status')),
|
||
'validation_passed': False}
|
||
if recovery.get('status') == 'REPAIRED':
|
||
from .native_repair import native_success_confirmed, select_repair_target
|
||
try:
|
||
confirmed = native_success_confirmed(select_repair_target(request, diagnosis), recovery)
|
||
except (ValueError, OSError):
|
||
confirmed = False
|
||
if not confirmed:
|
||
release = recovery.get('release')
|
||
recovery.update(status='UNCONFIRMED',
|
||
release={**(release if isinstance(release, dict) else {}), 'safe_to_continue': False},
|
||
detail='NATIVE_REPAIR_EVIDENCE_UNCONFIRMED: 原生成功记录不完整或矛盾,未重试校验')
|
||
if (recovery.get('status') == 'REPAIRED'
|
||
and recovery.get('release', {}).get('safe_to_continue') is True):
|
||
result = replace(original, recovery_attempts=(recovery,))
|
||
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
|
||
result = _run_verify_serialized(
|
||
replace(request, auto_repair=False, maintenance=False), progress=progress, flag_reader=flag_reader,
|
||
process_runner=process_runner, port_check=port_check,
|
||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt)
|
||
recovery['validation_evidence'] = str(result.evidence_path or '')
|
||
recovery['validation_passed'] = (result.status == 'PASS'
|
||
and result.library_flag == 'GREEN' and result.safe_to_continue)
|
||
if not recovery['validation_passed']:
|
||
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
|
||
if result.status == 'PASS':
|
||
result = replace(result, status='FAIL',
|
||
stderr=(result.stderr+'\nREPAIR_VALIDATION_UNCONFIRMED: 未取得校验 GREEN').strip())
|
||
else:
|
||
recovery['status'] = 'REPAIRED'
|
||
else:
|
||
# A failed local database must not block unrelated CBBs once
|
||
# every owner is released. Re-check target ownership here so
|
||
# an incomplete repair record cannot authorize the queue.
|
||
safe = False
|
||
if (recovery.get('status') in {'MISSING_HEALTHY_BACKUP', 'NATIVE_REPAIR_FAILED'}
|
||
and recovery.get('evidence', {}).get('failure_complete') is True
|
||
and recovery.get('release', {}).get('safe_to_continue') is True):
|
||
from .native_repair import select_repair_target
|
||
try:
|
||
target = select_repair_target(request, diagnosis)
|
||
safe = (Path(recovery.get('project', '')).resolve() == target.project
|
||
and Path(recovery.get('database', '')).resolve() == target.database)
|
||
except (ValueError, OSError):
|
||
pass
|
||
result = replace(result, safe_to_continue=safe, stderr=(result.stderr + '\n'
|
||
+ str(recovery.get('status', 'REPAIR_UNCONFIRMED')) + ': '
|
||
+ str(recovery.get('detail', '原生 Repair 未完成恢复'))).strip())
|
||
result = replace(result, recovery_attempts=(recovery,))
|
||
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
|
||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||
except Exception as error:
|
||
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status,
|
||
safe_to_continue=False,
|
||
stderr=(result.stderr+f'\nREPAIR_UNCONFIRMED: {error}').strip())
|
||
if result.evidence_path:
|
||
try:
|
||
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
|
||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||
except OSError:
|
||
pass # Keep the native/repair errors even if evidence storage fails.
|
||
release_confirmed = result.safe_to_continue
|
||
if not release_confirmed and result.evidence_path:
|
||
try:
|
||
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
|
||
release_confirmed = (release.get('worker_exited') is True
|
||
and release.get('safe_to_continue') is True and release.get('block') == request.block_name
|
||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve())
|
||
if (release.get('worker_exited') is True and not release.get('error')
|
||
and not release.get('close_errors') and not release.get('automation_release', {}).get('error')
|
||
and release.get('block') == request.block_name
|
||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
|
||
remaining_clients = tuple(pid for pid in release.get('remaining_pids', [])
|
||
if type(pid) is int and pid > 0)
|
||
except (OSError, ValueError, KeyError):
|
||
pass
|
||
if previous_release is not None:
|
||
result = replace(result, release_proof=previous_release)
|
||
if result.evidence_path:
|
||
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
|
||
return result
|
||
finally:
|
||
try:
|
||
if library_lock is not None:
|
||
library_lock.__exit__(None, None, None)
|
||
finally:
|
||
if acquired:
|
||
win32event.ReleaseMutex(mutex)
|
||
mutex.Close()
|
||
uninitialize_com(initialized)
|
||
|
||
|
||
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
|
||
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||
timing = RunTimings()
|
||
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
|
||
process_runner=process_runner, port_check=port_check,
|
||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
|
||
if not request.auto_repair and result.evidence_path:
|
||
try:
|
||
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
|
||
if (release.get('block') == request.block_name
|
||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
|
||
released = release.get('worker_exited') is True and release.get('safe_to_continue') is True
|
||
result = replace(result, safe_to_continue=released)
|
||
if not released and result.status == 'PASS':
|
||
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
|
||
except (OSError, ValueError, KeyError):
|
||
pass
|
||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
|
||
job_path = result.evidence_path.with_name('request.json')
|
||
if job_path.is_file():
|
||
timing.begin('failure_diagnostics')
|
||
try:
|
||
from .verify_diagnostics import capture_failure_logs_with_timeout
|
||
capture_failure_logs_with_timeout(job_path)
|
||
except Exception:
|
||
# Never replace the underlying Mentor/bridge/timeout failure.
|
||
pass
|
||
finally:
|
||
timing.end('failure_diagnostics')
|
||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}:
|
||
from .icdb_recovery import ERROR_CODE, RECOVERY_ACTION, parse_session_loss, recovery_detail
|
||
recovery = parse_session_loss(result.stderr)
|
||
if recovery is None and result.evidence_path:
|
||
try:
|
||
saved = json.loads(result.evidence_path.with_name('diagnostics').joinpath(
|
||
'icdb-recovery.json').read_text('utf-8'))
|
||
recovery = parse_session_loss(saved.get('native_message', ''))
|
||
except (OSError, ValueError, AttributeError):
|
||
pass
|
||
if recovery or ERROR_CODE.casefold() in result.stderr.casefold():
|
||
detail = result.stderr
|
||
if recovery and RECOVERY_ACTION not in detail:
|
||
detail = (detail + '\n' + recovery_detail(recovery)).strip()
|
||
result = replace(result, stderr=detail, error_category=ERROR_CODE)
|
||
if recovery and result.evidence_path:
|
||
try:
|
||
record_path = result.evidence_path.parent/'diagnostics/icdb-recovery.json'
|
||
record_path.parent.mkdir(parents=True, exist_ok=True)
|
||
if not record_path.is_file():
|
||
record_path.write_text(json.dumps(recovery, ensure_ascii=False, indent=2),
|
||
encoding='utf-8')
|
||
except OSError:
|
||
pass
|
||
durations, unfinished = timing.snapshot()
|
||
result = replace(result, timings_seconds=durations, incomplete_timing_phases=unfinished,
|
||
error_category=(result.error_category or classify_failure_category(result.stderr)
|
||
if result.status not in {"PASS", "SKIPPED"} else ""),
|
||
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
|
||
if result.status not in {"PASS", "SKIPPED"} else ""))
|
||
if result.evidence_path:
|
||
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
|
||
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
|
||
'block_name': request.block_name, 'status': result.status,
|
||
'unit': 'seconds', 'timings_seconds': durations,
|
||
'incomplete_timing_phases': unfinished,
|
||
'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)',
|
||
'notes': 'phase totals overlap action/call subphases; skipped phases are absent',
|
||
})
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'performance', 'block': request.block_name,
|
||
'timings_seconds': durations, 'incomplete_timing_phases': unfinished,
|
||
'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())})
|
||
if request.logging_context:
|
||
try:
|
||
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
|
||
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
|
||
except Exception:
|
||
logging.getLogger(__name__).warning('Operation logging failed', exc_info=True)
|
||
return result
|
||
|
||
|
||
def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||
"""Refresh one CBB and record its evidence.
|
||
|
||
``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``.
|
||
After successful execution, only IsVerifiedReuseBlock READY/raw=1 confirms
|
||
PASS/GREEN; Cert or icon observations never substitute for it. ``port_check`` is injectable so tests do
|
||
not depend on the host's port reservations.
|
||
"""
|
||
root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12])
|
||
try:
|
||
root.mkdir(parents=True, exist_ok=False)
|
||
except OSError as error:
|
||
return VerifyResult('FAIL', stderr=f'无法创建操作记录目录:{error}')
|
||
fingerprint = runtime_fingerprint(request.mentor_bin)
|
||
_write_optional_evidence(root/'runtime.json', fingerprint)
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'runtime_fingerprint', 'block': request.block_name,
|
||
**fingerprint})
|
||
result_path = root/'result.json'
|
||
runner = process_runner or (lambda command, timeout, cwd:
|
||
_run_process(command, timeout, cwd, evidence_root=root))
|
||
|
||
def finish(result):
|
||
# Only reclaim auxiliary backups after confirmed success. Failed runs
|
||
# retain every available original file for incident investigation.
|
||
try:
|
||
from .mentor_adapter import cleanup_evidence_backup
|
||
if result.status == 'PASS' and result.safe_to_continue:
|
||
cleanup_evidence_backup(root)
|
||
except Exception:
|
||
# Cleanup must never hide or change the verification result.
|
||
pass
|
||
return replace(result, evidence_path=result_path)
|
||
|
||
if stop is not None and stop.is_set():
|
||
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
|
||
timing.begin('preflight')
|
||
try:
|
||
project, pcb = preflight(request)
|
||
except Exception as error:
|
||
return finish(VerifyResult('BLOCKED', stderr=str(error)))
|
||
finally:
|
||
timing.end('preflight')
|
||
|
||
content_check_enabled = not request.diagnostic_only and bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
|
||
if content_check_enabled:
|
||
try:
|
||
executable = resolve_importer(request.mentor_bin)
|
||
except Exception as error:
|
||
return finish(VerifyResult('BLOCKED', stderr=str(error)))
|
||
command = build_content_check_command(request, executable)
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'content_check_start', 'block': request.block_name,
|
||
'detail': ' '.join(command)})
|
||
timing.begin('content_check')
|
||
try:
|
||
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
|
||
finally:
|
||
timing.end('content_check')
|
||
combined = '\n'.join(part for part in (stdout, stderr) if part)
|
||
report = contents_report_path(request.lmc_file, request.block_name)
|
||
try:
|
||
report.parent.mkdir(parents=True, exist_ok=True)
|
||
report.write_text(combined, encoding='utf-8')
|
||
except Exception as error:
|
||
stderr = f'{stderr}\n无法写入内容检查报告 {report}: {error}'
|
||
status = classify_output(exit_code, combined, timed_out=timed_out)
|
||
if status != 'PASS':
|
||
# A contents failure or an unusable DMS must block before touching the PCB.
|
||
return finish(VerifyResult(status if status in ('FAIL', 'TIMEOUT') else 'BLOCKED',
|
||
stderr=combined.strip() or f'内容检查结果未知(退出码 {exit_code})',
|
||
stdout=stdout, report_path=report, content_check_exit_code=exit_code,
|
||
safe_to_continue=not timed_out))
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
|
||
|
||
# Sampling cannot establish that the whole native port range is unusable.
|
||
from .mentor_adapter import icdb_ports_bindable
|
||
check_ports = port_check or icdb_ports_bindable
|
||
ports_ok, ports_detail = check_ports()
|
||
if not ports_ok:
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
|
||
if request.strict_port_check:
|
||
return finish(VerifyResult('BLOCKED', stderr=ports_detail))
|
||
|
||
job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name,
|
||
'diagnostic_only': request.diagnostic_only,
|
||
'supervisor_pid': os.getpid(),
|
||
'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds,
|
||
'mentor_bin': str(request.mentor_bin or ''),
|
||
'mapped_drive': str(request.mapped_drive or ''),
|
||
'mapped_remote_root': str(request.mapped_remote_root or ''),
|
||
'interactive_input': bool(request.interactive_input),
|
||
'native_lm_verify': bool(request.native_lm_verify),
|
||
'desktop': 'CbbVerify_' + uuid4().hex[:12],
|
||
'cancel_signal': str(root/'cancel-worker.signal'),
|
||
'start_gate': str(root/'start-worker.signal')}
|
||
consent = None
|
||
job_path = root/'request.json'
|
||
try:
|
||
if request.post_repair_proof is not None:
|
||
from .post_repair_consent import PostRepairProof, ConsentController, exact_prompt
|
||
if not isinstance(request.post_repair_proof, PostRepairProof):
|
||
return finish(VerifyResult('BLOCKED', stderr='POST_REPAIR_PROOF_REQUIRED'))
|
||
job['post_repair_consent'] = request.post_repair_proof.for_job(job)
|
||
consent = ConsentController(request.post_repair_proof, job)
|
||
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
|
||
except Exception as error:
|
||
return finish(VerifyResult('FAIL', stderr=str(error)))
|
||
events = root/'events.jsonl'
|
||
offset = 0
|
||
target_pid = None
|
||
launcher_pid = None
|
||
lm_pid = None
|
||
verify_invoked = False
|
||
invoke_returned = False
|
||
pcb_baseline = set()
|
||
last_observation = 0.0
|
||
last_post_verify_observation = {}
|
||
from .mentor_adapter import ActivityDeadline, pcb_startup_timeout_seconds
|
||
from .desktop_worker import DesktopWorker
|
||
from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend
|
||
from .dialog_models import DialogPhase
|
||
import win32event, win32con
|
||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||
lock_status = win32event.WaitForSingleObject(mutex, 0)
|
||
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
|
||
mutex.Close()
|
||
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。',
|
||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY'))
|
||
worker_launch_attempted = False
|
||
try:
|
||
timing.begin('worker_setup')
|
||
with (root/'worker.log').open('w', encoding='utf-8') as log:
|
||
worker_launch_attempted = True
|
||
if request.interactive_input:
|
||
process = DesktopWorker(
|
||
worker_command(job_path), None, interactive_input=True)
|
||
else:
|
||
process = DesktopWorker(worker_command(job_path), job['desktop'])
|
||
if hasattr(process, 'bind_cancellation'):
|
||
process.bind_cancellation(job['cancel_signal'])
|
||
deadline = ActivityDeadline(request.timeout_seconds)
|
||
last_activity_key = None
|
||
phase = DialogPhase.PCB_STARTING
|
||
restore_controller = None
|
||
dialog_agent = None
|
||
latest_snapshots = ()
|
||
if dialog_supervisor_factory is None:
|
||
if hasattr(process, 'start_companion'):
|
||
from .dialog_agent import DialogAgentClient
|
||
from .dialog_rules import default_rules_path, load_rules
|
||
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent', stop=stop)
|
||
dialog_supervisor = DialogSupervisor(
|
||
AgentDialogBackend(dialog_agent, pids=lambda: {
|
||
int(pid) for pid in (target_pid, lm_pid, launcher_pid) if pid}),
|
||
rules=load_rules(default_rules_path()),
|
||
)
|
||
else:
|
||
dialog_supervisor = DialogSupervisor(Win32DialogBackend(
|
||
lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
|
||
desktop=process.desktop,
|
||
))
|
||
else:
|
||
dialog_supervisor = dialog_supervisor_factory(
|
||
desktop=process.desktop,
|
||
pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
|
||
)
|
||
if stop is not None and stop.is_set():
|
||
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
|
||
Path(job['start_gate']).touch()
|
||
while process.poll() is None:
|
||
if stop is not None and stop.is_set():
|
||
_stop_worker(process)
|
||
return finish(VerifyResult('CANCELLED', stderr='已取消当前校验。',
|
||
safe_to_continue=False, error_stage=phase.value))
|
||
if events.exists():
|
||
with events.open(encoding='utf-8') as stream:
|
||
stream.seek(offset)
|
||
while line := stream.readline():
|
||
try:
|
||
event = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
break
|
||
timing.record(event)
|
||
if event.get('pcb_pid'): target_pid = event['pcb_pid']
|
||
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
|
||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||
stage = event.get('stage')
|
||
if stage == 'verify_invoke': verify_invoked = True
|
||
if stage == 'verify_invoke_returned': invoke_returned = True
|
||
if stage == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
|
||
phase = phase_for_event(event, phase)
|
||
activity_key = (stage, json.dumps(event.get('statuses'), sort_keys=True),
|
||
json.dumps(event.get('markers'), sort_keys=True),
|
||
event.get('action'), event.get('detail'))
|
||
# Repeated integration_status snapshots are normal
|
||
# polling, not progress. Only a changed state or
|
||
# a new stage/marker extends the inactivity window.
|
||
if activity_key != last_activity_key:
|
||
if stage in ('lm_started', 'lm_ready', 'verify_invoke',
|
||
'verify_invoke_returned', 'lm_session_ready',
|
||
'pcb_started', 'pcb_worker_started'):
|
||
deadline.timeout_seconds = pcb_startup_timeout_seconds(
|
||
request.timeout_seconds)
|
||
elif stage == 'identity_verified':
|
||
deadline.timeout_seconds = max(
|
||
0.1, float(request.timeout_seconds))
|
||
deadline.touch()
|
||
last_activity_key = activity_key
|
||
if progress: _notify_progress(progress, event)
|
||
offset = stream.tell()
|
||
if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5:
|
||
from .post_verify_diagnostics import observe_post_verify
|
||
last_observation = time.monotonic()
|
||
try:
|
||
observation = observe_post_verify(process.desktop, pcb, pcb_baseline, lm_pid)
|
||
except Exception as error:
|
||
observation = {'error': str(error)}
|
||
last_post_verify_observation = observation
|
||
if dialog_agent is not None and target_pid is None:
|
||
# The Library Manager Verify route starts its own window
|
||
# before it launches Expedition PCB. Do not feed that
|
||
# large MFC main window to the PCB dialog probe; it can
|
||
# exhaust the UIA observation deadline before Verify has
|
||
# even produced a target process.
|
||
from .dialog_supervisor import DialogPollResult
|
||
dialog_result = DialogPollResult(activity=False)
|
||
elif dialog_agent is not None:
|
||
dialog_result = dialog_supervisor.poll(phase)
|
||
elif dialog_supervisor_factory is not None:
|
||
try:
|
||
dialog_result = dialog_supervisor.poll(phase)
|
||
except TypeError:
|
||
# Compatibility for injected legacy test supervisors only.
|
||
dialog_result = dialog_supervisor.poll()
|
||
else:
|
||
dialog_result = dialog_supervisor.poll()
|
||
if (consent is not None and dialog_agent is not None
|
||
and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED'
|
||
and dialog_result.snapshot is not None and exact_prompt(dialog_result.snapshot)):
|
||
consent_snapshot = dialog_result.snapshot
|
||
if consent_snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}:
|
||
dialog_result = consent.handle(consent_snapshot, dialog_result.observed_snapshots,
|
||
target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid,
|
||
companion_pids=tuple(process.companion_pids), perform=dialog_supervisor.backend.perform)
|
||
if dialog_result.error_code == 'PCB_VERSION_CHOICE_REQUIRED' and dialog_result.snapshot is not None:
|
||
from .dialog_rules import archive_version_choice
|
||
choice_path = archive_version_choice(dialog_result.snapshot, root)
|
||
dialog_result = replace(dialog_result, detail=dialog_result.detail + f'\n版本候选与原生窗口证据:{choice_path}')
|
||
if (version_prompt is not None and consent is not None and dialog_agent is not None
|
||
and dialog_result.snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}):
|
||
from .pcb_restore_prompt import RestoreController
|
||
if restore_controller is None:
|
||
restore_controller = RestoreController(job)
|
||
dialog_result = restore_controller.handle(dialog_result.snapshot,
|
||
dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid,
|
||
worker_pid=process.pid, companion_pids=tuple(process.companion_pids),
|
||
show=version_prompt, perform=dialog_supervisor.backend.perform)
|
||
# Normal progress stays in memory. Only actions/errors and the
|
||
# last failure snapshots are persisted, not every elapsed tick.
|
||
latest_snapshots = dialog_result.observed_snapshots
|
||
if dialog_result.snapshot is not None and (
|
||
dialog_result.action_sent or dialog_result.error_code):
|
||
snapshot_record = (dialog_result.snapshot.to_record()
|
||
if hasattr(dialog_result.snapshot, 'to_record')
|
||
else dialog_result.snapshot.as_log_record())
|
||
record = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
**snapshot_record,
|
||
'action': dialog_result.action,
|
||
'rule_id': dialog_result.rule_id,
|
||
'channel': dialog_result.channel,
|
||
'accepted_control': list(dialog_result.accepted_controls),
|
||
'error_code': dialog_result.error_code,
|
||
'detail': dialog_result.detail,
|
||
}
|
||
with (root/'dialogs.jsonl').open('a', encoding='utf-8') as stream:
|
||
stream.write(json.dumps(record, ensure_ascii=False) + '\n')
|
||
if dialog_result.activity:
|
||
deadline.touch()
|
||
if dialog_result.error_code:
|
||
_write_failure_windows(root, latest_snapshots)
|
||
_write_process_tree(root, process, target_pid)
|
||
# A failed database load has no document to close. Give the
|
||
# worker time to normally close its owned empty PCB before
|
||
# recovery checks for remaining native database clients.
|
||
_stop_worker(process, timeout=40)
|
||
return finish(VerifyResult(
|
||
'FAIL',
|
||
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
|
||
error_stage=phase.value,
|
||
safe_to_continue=False,
|
||
))
|
||
if deadline.expired():
|
||
_write_failure_windows(root, latest_snapshots)
|
||
# Stop automation without killing PCB/iCDB during a write.
|
||
_write_process_tree(root, process, target_pid)
|
||
if last_post_verify_observation:
|
||
(root/'post_verify_observations.jsonl').write_text(json.dumps({
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'invoke_returned': invoke_returned,
|
||
**last_post_verify_observation,
|
||
}, ensure_ascii=False) + '\n', encoding='utf-8')
|
||
dispatch_failure = classify_verify_dispatch(
|
||
verify_invoked=verify_invoked,
|
||
invoke_returned=invoke_returned,
|
||
target_pid=target_pid,
|
||
observation=last_post_verify_observation,
|
||
)
|
||
_stop_worker(process)
|
||
if dispatch_failure is not None:
|
||
status, detail = dispatch_failure
|
||
if progress:
|
||
_notify_progress(progress, {
|
||
'stage': 'verify_dispatch_failed',
|
||
'block': request.block_name,
|
||
'detail': detail,
|
||
'observation': last_post_verify_observation,
|
||
})
|
||
return finish(VerifyResult(status, stderr=detail, safe_to_continue=False))
|
||
return finish(VerifyResult('TIMEOUT', stderr='操作超时,当前项未完成。', error_stage=phase.value, safe_to_continue=False))
|
||
time.sleep(.2)
|
||
worker_result = root/'outcome.json'
|
||
if not worker_result.exists():
|
||
return finish(VerifyResult('FAIL', stderr=f'执行器异常退出 ({process.returncode}),见 worker.log', safe_to_continue=False))
|
||
outcome = json.loads(worker_result.read_text(encoding='utf-8'))
|
||
if outcome.get('status') == 'PASS' and process.returncode != 0:
|
||
return finish(VerifyResult('FAIL', stderr='执行器异常退出,不能确认成功', safe_to_continue=False))
|
||
result = VerifyResult(**outcome)
|
||
except Exception as error:
|
||
if 'process' in locals() and process.poll() is None:
|
||
_stop_worker(process)
|
||
launch_release = getattr(error, 'worker_launch_release', None) if 'process' not in locals() else None
|
||
if not worker_launch_attempted:
|
||
launch_release = {'process_created': False, 'worker_exited': True,
|
||
'safe_to_continue': True, 'remaining_pids': []}
|
||
no_launch = (isinstance(launch_release, dict)
|
||
and launch_release.get('process_created') is False
|
||
and launch_release.get('worker_exited') is True
|
||
and launch_release.get('safe_to_continue') is True
|
||
and launch_release.get('remaining_pids') == [])
|
||
if no_launch:
|
||
_write_optional_evidence(root/'session-release.json', {
|
||
**launch_release, 'block': request.block_name, 'library': str(request.lmc_file),
|
||
'worker_pid': None})
|
||
status = 'CANCELLED' if stop is not None and stop.is_set() else 'FAIL'
|
||
return finish(VerifyResult(status, stderr=str(error), safe_to_continue=no_launch))
|
||
finally:
|
||
close_errors = []
|
||
worker_exited = False
|
||
automation_release = {'safe_to_continue': False}
|
||
if 'dialog_agent' in locals() and dialog_agent is not None:
|
||
try:
|
||
dialog_agent.close()
|
||
except Exception as error:
|
||
close_errors.append(str(error))
|
||
if 'process' in locals():
|
||
try:
|
||
worker_exited = process.poll() is not None
|
||
if not worker_exited:
|
||
worker_exited = _stop_worker(process)
|
||
if hasattr(process, 'automation_release_state'):
|
||
automation_release = process.automation_release_state()
|
||
else:
|
||
automation_release = {'safe_to_continue': worker_exited, 'remaining_pids': []}
|
||
process.close()
|
||
except Exception as error:
|
||
close_errors.append(str(error))
|
||
if 'process' in locals():
|
||
try:
|
||
if events.is_file():
|
||
with events.open(encoding='utf-8') as stream:
|
||
stream.seek(offset)
|
||
for line in stream:
|
||
try:
|
||
event = json.loads(line)
|
||
timing.record(event)
|
||
if event.get('pcb_pid'): target_pid = event['pcb_pid']
|
||
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
|
||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||
if event.get('stage') == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
offset = stream.tell()
|
||
release_result = _cleanup_session_processes(
|
||
root,
|
||
lm_pid=lm_pid,
|
||
target_pid=target_pid,
|
||
launcher_pid=launcher_pid,
|
||
pcb=pcb,
|
||
pcb_baseline=pcb_baseline,
|
||
progress=progress,
|
||
)
|
||
except Exception as error:
|
||
release_result = {'safe_to_continue': False, 'error': str(error)}
|
||
try:
|
||
from .icdb_ownership import unreleased_addresses
|
||
addresses = unreleased_addresses(request.lmc_file, project, pcb)
|
||
if addresses:
|
||
release_result = {**(release_result or {}), 'icdb_addresses': addresses}
|
||
except OSError as error:
|
||
close_errors.append(str(error))
|
||
if close_errors:
|
||
release_result = {**(release_result or {}), 'safe_to_continue': False,
|
||
'close_errors': close_errors}
|
||
if not worker_exited or not automation_release.get('safe_to_continue'):
|
||
release_result = {**(release_result or {}), 'safe_to_continue': False,
|
||
'automation_release': automation_release, 'worker_exited': worker_exited}
|
||
release_result['remaining_pids'] = sorted(set(release_result.get('remaining_pids', []))
|
||
| set(automation_release.get('remaining_pids', [])))
|
||
try:
|
||
(root/'session-release.json').write_text(json.dumps({
|
||
**(release_result or {}), 'block': request.block_name,
|
||
'library': str(request.lmc_file), 'worker_exited': worker_exited,
|
||
'worker_pid': getattr(process, 'pid', None)},
|
||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||
except OSError:
|
||
pass
|
||
if not release_result or not release_result.get('safe_to_continue'):
|
||
if 'result' in locals():
|
||
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False,
|
||
stderr=(result.stderr + '\nPCB_RELEASE_UNCONFIRMED: 未确认原生会话退出').strip())
|
||
recovery = {'desktop': job['desktop'], 'project': str(project),
|
||
'pcb': str(pcb), 'release': release_result,
|
||
'action': '停止批次;检查原生会话及数据库;不要删除锁文件或自动恢复。'}
|
||
try:
|
||
(root/'recovery-required.json').write_text(
|
||
json.dumps(recovery, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
except OSError:
|
||
pass # Evidence failure must not prevent mutex release.
|
||
# Drain the timing tail even when the worker exited before a poll.
|
||
# This is optional telemetry: an unreadable log must not mask failure
|
||
# or prevent releasing the serial-operation mutex.
|
||
try:
|
||
if events.is_file():
|
||
with events.open(encoding='utf-8') as stream:
|
||
stream.seek(offset)
|
||
for line in stream:
|
||
try:
|
||
timing.record(json.loads(line))
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
except OSError:
|
||
pass
|
||
finally:
|
||
win32event.ReleaseMutex(mutex)
|
||
mutex.Close()
|
||
|
||
if result.status == 'PASS' and not request.diagnostic_only:
|
||
timing.begin('library_flag')
|
||
bridge_records = []
|
||
def record_attempt(record):
|
||
bridge_records.append(record)
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
|
||
try:
|
||
probe = read_library_flag_with_retry(
|
||
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
|
||
request.block_name, on_attempt=record_attempt)
|
||
result = apply_post_verify_status(result, probe)
|
||
result = replace(result, bridge_attempts=tuple(bridge_records))
|
||
_write_optional_evidence(root/'bridge-status.json', bridge_records)
|
||
if progress:
|
||
_notify_progress(progress, {'stage': 'library_flag', 'block': request.block_name,
|
||
'library_flag': result.library_flag, 'detail': result.library_flag_detail})
|
||
finally:
|
||
timing.end('library_flag')
|
||
return finish(result)
|
||
|
||
|
||
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None):
|
||
"""Run serially and stop when database/session safety is unconfirmed."""
|
||
if max_workers != 1:
|
||
raise ValueError('Only serial Verify is supported for the shared Library Manager session')
|
||
if verifier is None:
|
||
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available) # noqa: E731
|
||
results = []
|
||
queue_stop = False
|
||
for request in requests:
|
||
if queue_stop or (stop is not None and stop.is_set()):
|
||
result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB')
|
||
else:
|
||
try:
|
||
result = verifier(request)
|
||
except Exception as error:
|
||
# An unexpected exception cannot establish database safety.
|
||
result = VerifyResult('FAIL', stderr=str(error), safe_to_continue=False)
|
||
results.append(result)
|
||
# Never start another writer after an unconfirmed native shutdown.
|
||
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'} or not result.safe_to_continue
|
||
if queue_stop and stop is not None:
|
||
stop.set()
|
||
if progress: _notify_progress(progress, request.block_name, result)
|
||
return results
|
||
|
||
|
||
def failure_report_path(batch_report):
|
||
batch_report = Path(batch_report)
|
||
return batch_report.with_name(batch_report.name.replace('batch-', 'failures-', 1))
|
||
|
||
|
||
def _report_field(value):
|
||
# One CBB per row; full native multi-line text remains in evidence files.
|
||
return str(value).replace('\r', r'\r').replace('\n', r'\n').replace('\t', r'\t')
|
||
|
||
|
||
def write_failure_report(requests, results, root, *, path=None):
|
||
requests, results = tuple(requests), tuple(results)
|
||
if len(requests) != len(results):
|
||
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
|
||
root = Path(root); root.mkdir(parents=True, exist_ok=True)
|
||
path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||
failed = [(req, result) for req, result in zip(requests, results)
|
||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}
|
||
and not (result.status in {'CHECKED', 'REPAIRED'} and result.safe_to_continue)]
|
||
passed = sum(result.status == 'PASS' for result in results)
|
||
checked = sum(result.status == 'CHECKED' and result.safe_to_continue for result in results)
|
||
repaired = sum(result.status == 'REPAIRED' and result.safe_to_continue for result in results)
|
||
skipped = len(results)-passed-len(failed)-checked-repaired
|
||
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped} 仅检查未修复: {checked} ICDB已修复待校验: {repaired}',
|
||
'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)']
|
||
if requests and all(req.maintenance for req in requests):
|
||
from .icdb_maintenance import maintenance_counts
|
||
counts = maintenance_counts(results)
|
||
lines[:2] = ['ICDB 检查异常清单',
|
||
f'总数: {len(results)} 未发现异常: {checked} 发现问题: {counts["issues_found"]} '
|
||
f'占用: {counts["in_use"]} 无法确认: {counts["unconfirmed"]} '
|
||
f'取消: {counts["cancelled"]} 未执行: {counts["skipped"]}']
|
||
for req, result in failed:
|
||
timings = ' '.join(f'{key}={value:.3f}s' +
|
||
('(unfinished)' if key in result.incomplete_timing_phases else '')
|
||
for key, value in result.timings_seconds.items())
|
||
lines.append(f'{req.block_name}\t{result.status}\t{result.error_category or classify_failure_category(result.stderr)}\t'
|
||
f'{result.error_stage or "unknown"}\t{result.library_flag}\t{_report_field(result.stderr or result.stdout.strip()[:200])}\t{result.evidence_path or ""}\t{timings}')
|
||
if not failed:
|
||
lines.append('本批次没有失败项。')
|
||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
|
||
return path
|
||
|
||
|
||
def write_batch_report(requests, results, root):
|
||
requests, results = tuple(requests), tuple(results)
|
||
if len(requests) != len(results):
|
||
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
|
||
root = Path(root); root.mkdir(parents=True, exist_ok=True)
|
||
path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||
lines = ['CBB Verify 批量结果',
|
||
'流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)']
|
||
if requests and all(req.maintenance for req in requests):
|
||
lines = ['ICDB 批量检查结果', '流程: 原生 PCB 打开检查;不执行 Repair、Restore 或 CBB 校验。']
|
||
for req, result in zip(requests, results):
|
||
detail = _report_field(result.stderr or result.stdout.strip()[:200])
|
||
seconds = ' '.join(f'{key}={value:.3f}s' +
|
||
('(unfinished)' if key in result.incomplete_timing_phases else '')
|
||
for key, value in result.timings_seconds.items())
|
||
lines.append(f'{req.block_name}\t{result.status}\t{result.library_flag}\t{detail}\t{result.evidence_path or ""}\t{seconds}')
|
||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
|
||
write_failure_report(requests, results, root, path=failure_report_path(path))
|
||
return path
|