feat: integrate native diagnostics and coordinated CBB transactions

This commit is contained in:
阿白叔 committed 2026-11-05 23:18:00 +08:00
1 parent 9e230e1a46
commit da67de026f
91 files changed
+14370 -435

No files matched your search

+325 -73
View File
@@ -14,6 +14,7 @@ native menu invocation happens on the worker's private desktop.
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
import json
import logging
import os
import platform
import re
@@ -32,6 +33,22 @@ ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE)
INVALID_NAME_CHARS = '\\/:*?"<>|'
def _notify_progress(callback, *args):
"""A failed observer cannot change native execution or release facts."""
if callback is not None:
try:
callback(*args)
except Exception:
logging.getLogger(__name__).warning('Progress notification failed', exc_info=True)
def _write_optional_evidence(path, value):
try:
path.write_text(json.dumps(value, ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except OSError:
logging.getLogger(__name__).warning('Could not write diagnostic evidence %s', path, exc_info=True)
@dataclass(frozen=True)
class VerifyRequest:
lmc_file: Path
@@ -56,13 +73,16 @@ class VerifyRequest:
# A reserved iCDB range is a host-wide prerequisite failure. Do not
# launch a PCB that is guaranteed to time out; batch orchestration will
# record this item and continue with the next one.
strict_port_check: bool = True
strict_port_check: bool = False
# A mapped drive can be invisible to an elevated/private worker on Win10.
# Preserve the parent session's mapping identity so the worker can restore
# it before launching Library Manager/PCB.
mapped_drive: str = ''
mapped_remote_root: str = ''
auto_repair: bool = True
auto_repair: bool = False
maintenance: bool = False
post_repair_proof: object | None = field(default=None, repr=False, compare=False)
diagnostic_only: bool = False
@dataclass(frozen=True)
@@ -132,7 +152,8 @@ def classify_output(exit_code: int | None, output: str, *, timed_out: bool = Fal
from .icdb_recovery import parse_session_loss
if parse_session_loss(output):
return 'FAIL'
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(output):
diagnostic_text = re.sub(r'\b0\s+errors?\b', '', output, flags=re.I)
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(diagnostic_text):
return 'PASS'
if ERROR_TEXT.search(output):
return 'FAIL'
@@ -154,7 +175,8 @@ def preflight(request):
if len(projects) != 1:
raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ')
project = absolute_preserving_drive(projects[0])
content = project.read_text(encoding='utf-8', errors='replace')
from .backup_recovery import _project_text
content = _project_text(project)
refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M)
if not refs:
raise ValueError(f'{name}: 工程没有 PCBDesignPath')
@@ -217,7 +239,17 @@ def _stop_worker(process, *, timeout=5):
try:
process.wait(timeout=timeout)
except Exception:
pass
try:
process.kill() # Automation clients only; never the native tree.
process.wait(timeout=5)
except Exception:
pass
try:
if hasattr(process, 'stop_automation_clients'):
process.stop_automation_clients()
return process.poll() is not None
except Exception:
return False
def _normalize_bridge_result(probe):
@@ -262,9 +294,16 @@ def read_library_flag_with_retry(reader, library, block_name, *, attempts=3,
def classify_failure_category(detail):
text = str(detail or '')
lower = text.casefold()
if 'pcb_version_choice_required' in lower:
return 'PCB_VERSION_CHOICE_REQUIRED'
from .icdb_recovery import ERROR_CODE, parse_session_loss
if ERROR_CODE.casefold() in lower or parse_session_loss(text):
return ERROR_CODE
if 'back annotation is not allowed on verified (or once verified) reusable blocks.' in lower:
# Preserve the native integration failure even when shutdown adds a
# secondary release warning. This category does not grant permission
# to reimport or change safe_to_continue.
return 'CBB_REIMPORT_REQUIRED'
if any(code in lower for code in (
'pcb_release_unconfirmed', 'icdb_release_unconfirmed',
'icdb_session_unconfirmed', 'pcb_session_active', 'database_recovery_required')):
@@ -470,7 +509,7 @@ def native_lm_verify_enabled():
}
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, launcher_pid=None,
progress=None):
"""Record native release or retention after automation has stopped."""
from .mentor_adapter import cleanup_verify_processes
@@ -489,12 +528,12 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
except OSError:
pass
if progress:
progress(event)
_notify_progress(progress, event)
return cleanup_verify_processes(
lm_pid,
target_pid,
None,
launcher_pid,
expected_pcb=pcb,
baseline_pids=pcb_baseline,
emit=emit,
@@ -502,22 +541,47 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None):
dialog_supervisor_factory=None, version_prompt=None, stop=None):
# Cover contents checking, native work and final state reads with one
# session-wide mutex. The worker-stage lock below is reentrant on Windows.
import win32event, win32con
from .com_runtime import initialize_com, uninitialize_com
initialized = initialize_com()
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
library_lock = None
release_confirmed = True
remaining_clients = ()
def clients_released():
if release_confirmed:
return True
from .library_lock import _supervisor_alive
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
try:
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。',
safe_to_continue=False)
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')
if request.maintenance:
from .maintenance_entry import run_icdb_check
return run_icdb_check(request, progress=progress, stop=stop)
from .library_lock import LibraryLock, LibraryBusy
import pywintypes
if Path(request.lmc_file).is_file():
try:
library_lock = LibraryLock(request.lmc_file, checkpoint=False,
safe_to_release=clients_released)
library_lock.__enter__()
except LibraryBusy as error:
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
except (OSError, pywintypes.error) as error:
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_ACCESS_FAILED')
release_confirmed = False
result = _run_verify_serialized(
request, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory)
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
if (request.auto_repair and result.status not in {'TIMEOUT', 'BLOCKED'}
and result.error_category == 'ICDB_REPAIR_REQUIRED'
and result.evidence_path):
@@ -525,14 +589,63 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
try:
diagnosis = json.loads((result.evidence_path.parent/'diagnostics/icdb-recovery.json').read_text('utf-8'))
original = result
snapshot = original.evidence_path.with_name('pre-repair-result.json')
snapshot.write_text(json.dumps(asdict(original), ensure_ascii=False, default=str, indent=2),
encoding='utf-8')
recovery = attempt_recovery(request, diagnosis, result.evidence_path.parent, progress=progress)
recovery = {**recovery, 'original_error': original.stderr,
'original_evidence': str(original.evidence_path)}
'original_evidence': str(original.evidence_path),
'original_result_snapshot': str(snapshot),
'original_stage': original.error_stage,
'native_status': recovery.get('native_status', recovery.get('status')),
'validation_passed': False}
if recovery.get('status') == 'REPAIRED':
from .native_repair import native_success_confirmed, select_repair_target
try:
confirmed = native_success_confirmed(select_repair_target(request, diagnosis), recovery)
except (ValueError, OSError):
confirmed = False
if not confirmed:
release = recovery.get('release')
recovery.update(status='UNCONFIRMED',
release={**(release if isinstance(release, dict) else {}), 'safe_to_continue': False},
detail='NATIVE_REPAIR_EVIDENCE_UNCONFIRMED: 原生成功记录不完整或矛盾,未重试校验')
if (recovery.get('status') == 'REPAIRED'
and recovery.get('release', {}).get('safe_to_continue') is True):
result = replace(original, recovery_attempts=(recovery,))
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
result = _run_verify_serialized(
replace(request, auto_repair=False), progress=progress, flag_reader=flag_reader,
replace(request, auto_repair=False, maintenance=False), progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory)
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt)
recovery['validation_evidence'] = str(result.evidence_path or '')
recovery['validation_passed'] = (result.status == 'PASS'
and result.library_flag == 'GREEN' and result.safe_to_continue)
if not recovery['validation_passed']:
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
if result.status == 'PASS':
result = replace(result, status='FAIL',
stderr=(result.stderr+'\nREPAIR_VALIDATION_UNCONFIRMED: 未取得校验 GREEN').strip())
else:
recovery['status'] = 'REPAIRED'
else:
# A failed local database must not block unrelated CBBs once
# every owner is released. Re-check target ownership here so
# an incomplete repair record cannot authorize the queue.
safe = False
if (recovery.get('status') in {'MISSING_HEALTHY_BACKUP', 'NATIVE_REPAIR_FAILED'}
and recovery.get('evidence', {}).get('failure_complete') is True
and recovery.get('release', {}).get('safe_to_continue') is True):
from .native_repair import select_repair_target
try:
target = select_repair_target(request, diagnosis)
safe = (Path(recovery.get('project', '')).resolve() == target.project
and Path(recovery.get('database', '')).resolve() == target.database)
except (ValueError, OSError):
pass
result = replace(result, safe_to_continue=safe, stderr=(result.stderr + '\n'
+ str(recovery.get('status', 'REPAIR_UNCONFIRMED')) + ': '
+ str(recovery.get('detail', '原生 Repair 未完成恢复'))).strip())
result = replace(result, recovery_attempts=(recovery,))
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
@@ -540,20 +653,57 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status,
safe_to_continue=False,
stderr=(result.stderr+f'\nREPAIR_UNCONFIRMED: {error}').strip())
if result.evidence_path:
try:
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except OSError:
pass # Keep the native/repair errors even if evidence storage fails.
release_confirmed = result.safe_to_continue
if not release_confirmed and result.evidence_path:
try:
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
release_confirmed = (release.get('worker_exited') is True
and release.get('safe_to_continue') is True and release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve())
if (release.get('worker_exited') is True and not release.get('error')
and not release.get('close_errors') and not release.get('automation_release', {}).get('error')
and release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
remaining_clients = tuple(pid for pid in release.get('remaining_pids', [])
if type(pid) is int and pid > 0)
except (OSError, ValueError, KeyError):
pass
return result
finally:
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
try:
if library_lock is not None:
library_lock.__exit__(None, None, None)
finally:
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
uninitialize_com(initialized)
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
port_check=None, dialog_supervisor_factory=None):
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
timing = RunTimings()
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
dialog_supervisor_factory=dialog_supervisor_factory)
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'} and result.evidence_path:
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
if not request.auto_repair and result.evidence_path:
try:
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
if (release.get('block') == request.block_name
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
released = release.get('worker_exited') is True and release.get('safe_to_continue') is True
result = replace(result, safe_to_continue=released)
if not released and result.status == 'PASS':
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
except (OSError, ValueError, KeyError):
pass
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
job_path = result.evidence_path.with_name('request.json')
if job_path.is_file():
timing.begin('failure_diagnostics')
@@ -579,8 +729,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
detail = result.stderr
if recovery and RECOVERY_ACTION not in detail:
detail = (detail + '\n' + recovery_detail(recovery)).strip()
result = replace(result, safe_to_continue=False, stderr=detail,
error_category=ERROR_CODE)
result = replace(result, stderr=detail, error_category=ERROR_CODE)
if recovery and result.evidence_path:
try:
record_path = result.evidence_path.parent/'diagnostics/icdb-recovery.json'
@@ -597,28 +746,29 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
if result.status not in {"PASS", "SKIPPED"} else ""))
if result.evidence_path:
result.evidence_path.write_text(json.dumps(
{'block_name': request.block_name, **asdict(result)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
result.evidence_path.with_name('performance.json').write_text(json.dumps({
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
'block_name': request.block_name, 'status': result.status,
'unit': 'seconds', 'timings_seconds': durations,
'incomplete_timing_phases': unfinished,
'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)',
'notes': 'phase totals overlap action/call subphases; skipped phases are absent',
}, ensure_ascii=False, indent=2), encoding='utf-8')
})
if progress:
progress({'stage': 'performance', 'block': request.block_name,
_notify_progress(progress, {'stage': 'performance', 'block': request.block_name,
'timings_seconds': durations, 'incomplete_timing_phases': unfinished,
'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())})
if request.logging_context:
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
try:
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
except Exception:
logging.getLogger(__name__).warning('Operation logging failed', exc_info=True)
return result
def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None):
dialog_supervisor_factory=None, version_prompt=None, stop=None):
"""Refresh one CBB and record its evidence.
``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``.
@@ -627,12 +777,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
not depend on the host's port reservations.
"""
root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12])
root.mkdir(parents=True, exist_ok=False)
try:
root.mkdir(parents=True, exist_ok=False)
except OSError as error:
return VerifyResult('FAIL', stderr=f'无法创建操作记录目录:{error}')
fingerprint = runtime_fingerprint(request.mentor_bin)
(root/'runtime.json').write_text(
json.dumps(fingerprint, ensure_ascii=False, indent=2), encoding='utf-8')
_write_optional_evidence(root/'runtime.json', fingerprint)
if progress:
progress({'stage': 'runtime_fingerprint', 'block': request.block_name,
_notify_progress(progress, {'stage': 'runtime_fingerprint', 'block': request.block_name,
**fingerprint})
result_path = root/'result.json'
runner = process_runner or (lambda command, timeout, cwd:
@@ -650,6 +802,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
pass
return replace(result, evidence_path=result_path)
if stop is not None and stop.is_set():
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
timing.begin('preflight')
try:
project, pcb = preflight(request)
@@ -658,18 +812,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
finally:
timing.end('preflight')
from .icdb_ownership import unreleased_addresses
try:
addresses = unreleased_addresses(request.lmc_file, project, pcb)
if addresses:
return finish(VerifyResult('BLOCKED', safe_to_continue=False,
stderr='ICDB_SESSION_UNCONFIRMED: 原生数据库所有权未释放,未开始内容检查或校验:'
+ '; '.join(addresses)))
except OSError as error:
return finish(VerifyResult('BLOCKED', safe_to_continue=False,
stderr=f'ICDB_SESSION_UNCONFIRMED: 无法确认数据库所有权:{error}'))
content_check_enabled = bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
content_check_enabled = not request.diagnostic_only and bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
if content_check_enabled:
try:
executable = resolve_importer(request.mentor_bin)
@@ -677,7 +820,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
return finish(VerifyResult('BLOCKED', stderr=str(error)))
command = build_content_check_command(request, executable)
if progress:
progress({'stage': 'content_check_start', 'block': request.block_name,
_notify_progress(progress, {'stage': 'content_check_start', 'block': request.block_name,
'detail': ' '.join(command)})
timing.begin('content_check')
try:
@@ -699,20 +842,20 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
stdout=stdout, report_path=report, content_check_exit_code=exit_code,
safe_to_continue=not timed_out))
if progress:
progress({'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
_notify_progress(progress, {'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
# The design server needs a port in 10000-10100; a system reservation there
# blocks every item, so fail fast with the fix instead of timing out.
# Sampling cannot establish that the whole native port range is unusable.
from .mentor_adapter import icdb_ports_bindable
check_ports = port_check or icdb_ports_bindable
ports_ok, ports_detail = check_ports()
if not ports_ok:
if progress:
progress({'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
_notify_progress(progress, {'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
if request.strict_port_check:
return finish(VerifyResult('BLOCKED', stderr=ports_detail))
job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name,
'diagnostic_only': request.diagnostic_only,
'supervisor_pid': os.getpid(),
'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds,
'mentor_bin': str(request.mentor_bin or ''),
@@ -723,11 +866,22 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
'desktop': 'CbbVerify_' + uuid4().hex[:12],
'cancel_signal': str(root/'cancel-worker.signal'),
'start_gate': str(root/'start-worker.signal')}
consent = None
job_path = root/'request.json'
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
try:
if request.post_repair_proof is not None:
from .post_repair_consent import PostRepairProof, ConsentController, exact_prompt
if not isinstance(request.post_repair_proof, PostRepairProof):
return finish(VerifyResult('BLOCKED', stderr='POST_REPAIR_PROOF_REQUIRED'))
job['post_repair_consent'] = request.post_repair_proof.for_job(job)
consent = ConsentController(request.post_repair_proof, job)
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
except Exception as error:
return finish(VerifyResult('FAIL', stderr=str(error)))
events = root/'events.jsonl'
offset = 0
target_pid = None
launcher_pid = None
lm_pid = None
verify_invoked = False
invoke_returned = False
@@ -743,10 +897,13 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
lock_status = win32event.WaitForSingleObject(mutex, 0)
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
mutex.Close()
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。', safe_to_continue=False))
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。',
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY'))
worker_launch_attempted = False
try:
timing.begin('worker_setup')
with (root/'worker.log').open('w', encoding='utf-8') as log:
worker_launch_attempted = True
if request.interactive_input:
process = DesktopWorker(
worker_command(job_path), None, interactive_input=True)
@@ -757,15 +914,17 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
deadline = ActivityDeadline(request.timeout_seconds)
last_activity_key = None
phase = DialogPhase.PCB_STARTING
restore_controller = None
dialog_agent = None
latest_snapshots = ()
if dialog_supervisor_factory is None:
if hasattr(process, 'start_companion'):
from .dialog_agent import DialogAgentClient
from .dialog_rules import default_rules_path, load_rules
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent')
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent', stop=stop)
dialog_supervisor = DialogSupervisor(
AgentDialogBackend(dialog_agent),
AgentDialogBackend(dialog_agent, pids=lambda: {
int(pid) for pid in (target_pid, lm_pid, launcher_pid) if pid}),
rules=load_rules(default_rules_path()),
)
else:
@@ -778,8 +937,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
desktop=process.desktop,
pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
)
if stop is not None and stop.is_set():
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
Path(job['start_gate']).touch()
while process.poll() is None:
if stop is not None and stop.is_set():
_stop_worker(process)
return finish(VerifyResult('CANCELLED', stderr='已取消当前校验。',
safe_to_continue=False, error_stage=phase.value))
if events.exists():
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
@@ -790,6 +955,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
break
timing.record(event)
if event.get('pcb_pid'): target_pid = event['pcb_pid']
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
if event.get('lm_pid'): lm_pid = event['lm_pid']
stage = event.get('stage')
if stage == 'verify_invoke': verify_invoked = True
@@ -813,7 +979,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
0.1, float(request.timeout_seconds))
deadline.touch()
last_activity_key = activity_key
if progress: progress(event)
if progress: _notify_progress(progress, event)
offset = stream.tell()
if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5:
from .post_verify_diagnostics import observe_post_verify
@@ -841,6 +1007,27 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
dialog_result = dialog_supervisor.poll()
else:
dialog_result = dialog_supervisor.poll()
if (consent is not None and dialog_agent is not None
and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED'
and dialog_result.snapshot is not None and exact_prompt(dialog_result.snapshot)):
consent_snapshot = dialog_result.snapshot
if consent_snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}:
dialog_result = consent.handle(consent_snapshot, dialog_result.observed_snapshots,
target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid,
companion_pids=tuple(process.companion_pids), perform=dialog_supervisor.backend.perform)
if dialog_result.error_code == 'PCB_VERSION_CHOICE_REQUIRED' and dialog_result.snapshot is not None:
from .dialog_rules import archive_version_choice
choice_path = archive_version_choice(dialog_result.snapshot, root)
dialog_result = replace(dialog_result, detail=dialog_result.detail + f'\n版本候选与原生窗口证据:{choice_path}')
if (version_prompt is not None and consent is not None and dialog_agent is not None
and dialog_result.snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}):
from .pcb_restore_prompt import RestoreController
if restore_controller is None:
restore_controller = RestoreController(job)
dialog_result = restore_controller.handle(dialog_result.snapshot,
dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid,
worker_pid=process.pid, companion_pids=tuple(process.companion_pids),
show=version_prompt, perform=dialog_supervisor.backend.perform)
# Normal progress stays in memory. Only actions/errors and the
# last failure snapshots are persisted, not every elapsed tick.
latest_snapshots = dialog_result.observed_snapshots
@@ -866,7 +1053,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
if dialog_result.error_code:
_write_failure_windows(root, latest_snapshots)
_write_process_tree(root, process, target_pid)
_stop_worker(process)
# A failed database load has no document to close. Give the
# worker time to normally close its owned empty PCB before
# recovery checks for remaining native database clients.
_stop_worker(process, timeout=40)
return finish(VerifyResult(
'FAIL',
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
@@ -893,14 +1083,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
if dispatch_failure is not None:
status, detail = dispatch_failure
if progress:
progress({
_notify_progress(progress, {
'stage': 'verify_dispatch_failed',
'block': request.block_name,
'detail': detail,
'observation': last_post_verify_observation,
})
return finish(VerifyResult(status, stderr=detail, safe_to_continue=False))
return finish(VerifyResult('TIMEOUT', stderr='操作超时;已停止自动化并保留原生会话,停止批次,请检查恢复记录。', error_stage=phase.value, safe_to_continue=False))
return finish(VerifyResult('TIMEOUT', stderr='操作超时,当前项未完成。', error_stage=phase.value, safe_to_continue=False))
time.sleep(.2)
worker_result = root/'outcome.json'
if not worker_result.exists():
@@ -912,9 +1102,25 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
except Exception as error:
if 'process' in locals() and process.poll() is None:
_stop_worker(process)
return finish(VerifyResult('FAIL', stderr=str(error), safe_to_continue=False))
launch_release = getattr(error, 'worker_launch_release', None) if 'process' not in locals() else None
if not worker_launch_attempted:
launch_release = {'process_created': False, 'worker_exited': True,
'safe_to_continue': True, 'remaining_pids': []}
no_launch = (isinstance(launch_release, dict)
and launch_release.get('process_created') is False
and launch_release.get('worker_exited') is True
and launch_release.get('safe_to_continue') is True
and launch_release.get('remaining_pids') == [])
if no_launch:
_write_optional_evidence(root/'session-release.json', {
**launch_release, 'block': request.block_name, 'library': str(request.lmc_file),
'worker_pid': None})
status = 'CANCELLED' if stop is not None and stop.is_set() else 'FAIL'
return finish(VerifyResult(status, stderr=str(error), safe_to_continue=no_launch))
finally:
close_errors = []
worker_exited = False
automation_release = {'safe_to_continue': False}
if 'dialog_agent' in locals() and dialog_agent is not None:
try:
dialog_agent.close()
@@ -922,15 +1128,37 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
close_errors.append(str(error))
if 'process' in locals():
try:
worker_exited = process.poll() is not None
if not worker_exited:
worker_exited = _stop_worker(process)
if hasattr(process, 'automation_release_state'):
automation_release = process.automation_release_state()
else:
automation_release = {'safe_to_continue': worker_exited, 'remaining_pids': []}
process.close()
except Exception as error:
close_errors.append(str(error))
if 'process' in locals():
try:
if events.is_file():
with events.open(encoding='utf-8') as stream:
stream.seek(offset)
for line in stream:
try:
event = json.loads(line)
timing.record(event)
if event.get('pcb_pid'): target_pid = event['pcb_pid']
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
if event.get('lm_pid'): lm_pid = event['lm_pid']
if event.get('stage') == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
except (json.JSONDecodeError, ValueError):
continue
offset = stream.tell()
release_result = _cleanup_session_processes(
root,
lm_pid=lm_pid,
target_pid=target_pid,
launcher_pid=launcher_pid,
pcb=pcb,
pcb_baseline=pcb_baseline,
progress=progress,
@@ -941,16 +1169,28 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
from .icdb_ownership import unreleased_addresses
addresses = unreleased_addresses(request.lmc_file, project, pcb)
if addresses:
release_result = {**(release_result or {}), 'safe_to_continue': False,
'icdb_address': addresses[0], 'icdb_addresses': addresses}
release_result = {**(release_result or {}), 'icdb_addresses': addresses}
except OSError as error:
close_errors.append(str(error))
if close_errors:
release_result = {**(release_result or {}), 'safe_to_continue': False,
'close_errors': close_errors}
if not worker_exited or not automation_release.get('safe_to_continue'):
release_result = {**(release_result or {}), 'safe_to_continue': False,
'automation_release': automation_release, 'worker_exited': worker_exited}
release_result['remaining_pids'] = sorted(set(release_result.get('remaining_pids', []))
| set(automation_release.get('remaining_pids', [])))
try:
(root/'session-release.json').write_text(json.dumps({
**(release_result or {}), 'block': request.block_name,
'library': str(request.lmc_file), 'worker_exited': worker_exited,
'worker_pid': getattr(process, 'pid', None)},
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
except OSError:
pass
if not release_result or not release_result.get('safe_to_continue'):
if 'result' in locals():
result = replace(result, status='FAIL', safe_to_continue=False,
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False,
stderr=(result.stderr + '\nPCB_RELEASE_UNCONFIRMED: 未确认原生会话退出').strip())
recovery = {'desktop': job['desktop'], 'project': str(project),
'pcb': str(pcb), 'release': release_result,
@@ -978,22 +1218,22 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
win32event.ReleaseMutex(mutex)
mutex.Close()
if result.status == 'PASS':
if result.status == 'PASS' and not request.diagnostic_only:
timing.begin('library_flag')
bridge_records = []
def record_attempt(record):
bridge_records.append(record)
if progress:
progress({'stage': 'library_flag_probe', 'block': request.block_name, **record})
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
try:
probe = read_library_flag_with_retry(
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
request.block_name, on_attempt=record_attempt)
result = apply_post_verify_status(result, probe)
result = replace(result, bridge_attempts=tuple(bridge_records))
(root/'bridge-status.json').write_text(json.dumps(bridge_records, ensure_ascii=False, indent=2), encoding='utf-8')
_write_optional_evidence(root/'bridge-status.json', bridge_records)
if progress:
progress({'stage': 'library_flag', 'block': request.block_name,
_notify_progress(progress, {'stage': 'library_flag', 'block': request.block_name,
'library_flag': result.library_flag, 'detail': result.library_flag_detail})
finally:
timing.end('library_flag')
@@ -1019,10 +1259,10 @@ def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=Non
result = VerifyResult('FAIL', stderr=str(error), safe_to_continue=False)
results.append(result)
# Never start another writer after an unconfirmed native shutdown.
queue_stop = queue_stop or result.status == 'QUEUE_STOP' or not result.safe_to_continue
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'} or not result.safe_to_continue
if queue_stop and stop is not None:
stop.set()
if progress: progress(request.block_name, result)
if progress: _notify_progress(progress, request.block_name, result)
return results
@@ -1043,11 +1283,21 @@ def write_failure_report(requests, results, root, *, path=None):
root = Path(root); root.mkdir(parents=True, exist_ok=True)
path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
failed = [(req, result) for req, result in zip(requests, results)
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}]
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}
and not (result.status in {'CHECKED', 'REPAIRED'} and result.safe_to_continue)]
passed = sum(result.status == 'PASS' for result in results)
skipped = len(results)-passed-len(failed)
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped}',
checked = sum(result.status == 'CHECKED' and result.safe_to_continue for result in results)
repaired = sum(result.status == 'REPAIRED' and result.safe_to_continue for result in results)
skipped = len(results)-passed-len(failed)-checked-repaired
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped} 仅检查未修复: {checked} ICDB已修复待校验: {repaired}',
'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)']
if requests and all(req.maintenance for req in requests):
from .icdb_maintenance import maintenance_counts
counts = maintenance_counts(results)
lines[:2] = ['ICDB 检查异常清单',
f'总数: {len(results)} 未发现异常: {checked} 发现问题: {counts["issues_found"]} '
f'占用: {counts["in_use"]} 无法确认: {counts["unconfirmed"]} '
f'取消: {counts["cancelled"]} 未执行: {counts["skipped"]}']
for req, result in failed:
timings = ' '.join(f'{key}={value:.3f}s' +
('(unfinished)' if key in result.incomplete_timing_phases else '')
@@ -1068,6 +1318,8 @@ def write_batch_report(requests, results, root):
path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
lines = ['CBB Verify 批量结果',
'流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)']
if requests and all(req.maintenance for req in requests):
lines = ['ICDB 批量检查结果', '流程: 原生 PCB 打开检查;不执行 Repair、Restore 或 CBB 校验。']
for req, result in zip(requests, results):
detail = _report_field(result.stderr or result.stdout.strip()[:200])
seconds = ' '.join(f'{key}={value:.3f}s' +