feat: integrate native diagnostics and coordinated CBB transactions
This commit is contained in:
1 parent
9e230e1a46
commit
da67de026f
91 files changed
+14370
-435
No files matched your search
@@ -14,6 +14,7 @@ native menu invocation happens on the worker's private desktop.
|
||||
from dataclasses import asdict, dataclass, field, replace
|
||||
from pathlib import Path
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
@@ -32,6 +33,22 @@ ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE)
|
||||
INVALID_NAME_CHARS = '\\/:*?"<>|'
|
||||
|
||||
|
||||
def _notify_progress(callback, *args):
|
||||
"""A failed observer cannot change native execution or release facts."""
|
||||
if callback is not None:
|
||||
try:
|
||||
callback(*args)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning('Progress notification failed', exc_info=True)
|
||||
|
||||
|
||||
def _write_optional_evidence(path, value):
|
||||
try:
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||||
except OSError:
|
||||
logging.getLogger(__name__).warning('Could not write diagnostic evidence %s', path, exc_info=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifyRequest:
|
||||
lmc_file: Path
|
||||
@@ -56,13 +73,16 @@ class VerifyRequest:
|
||||
# A reserved iCDB range is a host-wide prerequisite failure. Do not
|
||||
# launch a PCB that is guaranteed to time out; batch orchestration will
|
||||
# record this item and continue with the next one.
|
||||
strict_port_check: bool = True
|
||||
strict_port_check: bool = False
|
||||
# A mapped drive can be invisible to an elevated/private worker on Win10.
|
||||
# Preserve the parent session's mapping identity so the worker can restore
|
||||
# it before launching Library Manager/PCB.
|
||||
mapped_drive: str = ''
|
||||
mapped_remote_root: str = ''
|
||||
auto_repair: bool = True
|
||||
auto_repair: bool = False
|
||||
maintenance: bool = False
|
||||
post_repair_proof: object | None = field(default=None, repr=False, compare=False)
|
||||
diagnostic_only: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -132,7 +152,8 @@ def classify_output(exit_code: int | None, output: str, *, timed_out: bool = Fal
|
||||
from .icdb_recovery import parse_session_loss
|
||||
if parse_session_loss(output):
|
||||
return 'FAIL'
|
||||
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(output):
|
||||
diagnostic_text = re.sub(r'\b0\s+errors?\b', '', output, flags=re.I)
|
||||
if exit_code == 0 and SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(diagnostic_text):
|
||||
return 'PASS'
|
||||
if ERROR_TEXT.search(output):
|
||||
return 'FAIL'
|
||||
@@ -154,7 +175,8 @@ def preflight(request):
|
||||
if len(projects) != 1:
|
||||
raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ')
|
||||
project = absolute_preserving_drive(projects[0])
|
||||
content = project.read_text(encoding='utf-8', errors='replace')
|
||||
from .backup_recovery import _project_text
|
||||
content = _project_text(project)
|
||||
refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M)
|
||||
if not refs:
|
||||
raise ValueError(f'{name}: 工程没有 PCBDesignPath')
|
||||
@@ -217,7 +239,17 @@ def _stop_worker(process, *, timeout=5):
|
||||
try:
|
||||
process.wait(timeout=timeout)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
process.kill() # Automation clients only; never the native tree.
|
||||
process.wait(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if hasattr(process, 'stop_automation_clients'):
|
||||
process.stop_automation_clients()
|
||||
return process.poll() is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_bridge_result(probe):
|
||||
@@ -262,9 +294,16 @@ def read_library_flag_with_retry(reader, library, block_name, *, attempts=3,
|
||||
def classify_failure_category(detail):
|
||||
text = str(detail or '')
|
||||
lower = text.casefold()
|
||||
if 'pcb_version_choice_required' in lower:
|
||||
return 'PCB_VERSION_CHOICE_REQUIRED'
|
||||
from .icdb_recovery import ERROR_CODE, parse_session_loss
|
||||
if ERROR_CODE.casefold() in lower or parse_session_loss(text):
|
||||
return ERROR_CODE
|
||||
if 'back annotation is not allowed on verified (or once verified) reusable blocks.' in lower:
|
||||
# Preserve the native integration failure even when shutdown adds a
|
||||
# secondary release warning. This category does not grant permission
|
||||
# to reimport or change safe_to_continue.
|
||||
return 'CBB_REIMPORT_REQUIRED'
|
||||
if any(code in lower for code in (
|
||||
'pcb_release_unconfirmed', 'icdb_release_unconfirmed',
|
||||
'icdb_session_unconfirmed', 'pcb_session_active', 'database_recovery_required')):
|
||||
@@ -470,7 +509,7 @@ def native_lm_verify_enabled():
|
||||
}
|
||||
|
||||
|
||||
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
|
||||
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, launcher_pid=None,
|
||||
progress=None):
|
||||
"""Record native release or retention after automation has stopped."""
|
||||
from .mentor_adapter import cleanup_verify_processes
|
||||
@@ -489,12 +528,12 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
|
||||
except OSError:
|
||||
pass
|
||||
if progress:
|
||||
progress(event)
|
||||
_notify_progress(progress, event)
|
||||
|
||||
return cleanup_verify_processes(
|
||||
lm_pid,
|
||||
target_pid,
|
||||
None,
|
||||
launcher_pid,
|
||||
expected_pcb=pcb,
|
||||
baseline_pids=pcb_baseline,
|
||||
emit=emit,
|
||||
@@ -502,22 +541,47 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
|
||||
|
||||
|
||||
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||||
dialog_supervisor_factory=None):
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
# Cover contents checking, native work and final state reads with one
|
||||
# session-wide mutex. The worker-stage lock below is reentrant on Windows.
|
||||
import win32event, win32con
|
||||
from .com_runtime import initialize_com, uninitialize_com
|
||||
initialized = initialize_com()
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
library_lock = None
|
||||
release_confirmed = True
|
||||
remaining_clients = ()
|
||||
def clients_released():
|
||||
if release_confirmed:
|
||||
return True
|
||||
from .library_lock import _supervisor_alive
|
||||
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
|
||||
try:
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。',
|
||||
safe_to_continue=False)
|
||||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')
|
||||
if request.maintenance:
|
||||
from .maintenance_entry import run_icdb_check
|
||||
return run_icdb_check(request, progress=progress, stop=stop)
|
||||
from .library_lock import LibraryLock, LibraryBusy
|
||||
import pywintypes
|
||||
if Path(request.lmc_file).is_file():
|
||||
try:
|
||||
library_lock = LibraryLock(request.lmc_file, checkpoint=False,
|
||||
safe_to_release=clients_released)
|
||||
library_lock.__enter__()
|
||||
except LibraryBusy as error:
|
||||
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
|
||||
except (OSError, pywintypes.error) as error:
|
||||
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_ACCESS_FAILED')
|
||||
release_confirmed = False
|
||||
result = _run_verify_serialized(
|
||||
request, progress=progress, flag_reader=flag_reader,
|
||||
process_runner=process_runner, port_check=port_check,
|
||||
dialog_supervisor_factory=dialog_supervisor_factory)
|
||||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
|
||||
if (request.auto_repair and result.status not in {'TIMEOUT', 'BLOCKED'}
|
||||
and result.error_category == 'ICDB_REPAIR_REQUIRED'
|
||||
and result.evidence_path):
|
||||
@@ -525,14 +589,63 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
try:
|
||||
diagnosis = json.loads((result.evidence_path.parent/'diagnostics/icdb-recovery.json').read_text('utf-8'))
|
||||
original = result
|
||||
snapshot = original.evidence_path.with_name('pre-repair-result.json')
|
||||
snapshot.write_text(json.dumps(asdict(original), ensure_ascii=False, default=str, indent=2),
|
||||
encoding='utf-8')
|
||||
recovery = attempt_recovery(request, diagnosis, result.evidence_path.parent, progress=progress)
|
||||
recovery = {**recovery, 'original_error': original.stderr,
|
||||
'original_evidence': str(original.evidence_path)}
|
||||
'original_evidence': str(original.evidence_path),
|
||||
'original_result_snapshot': str(snapshot),
|
||||
'original_stage': original.error_stage,
|
||||
'native_status': recovery.get('native_status', recovery.get('status')),
|
||||
'validation_passed': False}
|
||||
if recovery.get('status') == 'REPAIRED':
|
||||
from .native_repair import native_success_confirmed, select_repair_target
|
||||
try:
|
||||
confirmed = native_success_confirmed(select_repair_target(request, diagnosis), recovery)
|
||||
except (ValueError, OSError):
|
||||
confirmed = False
|
||||
if not confirmed:
|
||||
release = recovery.get('release')
|
||||
recovery.update(status='UNCONFIRMED',
|
||||
release={**(release if isinstance(release, dict) else {}), 'safe_to_continue': False},
|
||||
detail='NATIVE_REPAIR_EVIDENCE_UNCONFIRMED: 原生成功记录不完整或矛盾,未重试校验')
|
||||
if (recovery.get('status') == 'REPAIRED'
|
||||
and recovery.get('release', {}).get('safe_to_continue') is True):
|
||||
result = replace(original, recovery_attempts=(recovery,))
|
||||
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
|
||||
result = _run_verify_serialized(
|
||||
replace(request, auto_repair=False), progress=progress, flag_reader=flag_reader,
|
||||
replace(request, auto_repair=False, maintenance=False), progress=progress, flag_reader=flag_reader,
|
||||
process_runner=process_runner, port_check=port_check,
|
||||
dialog_supervisor_factory=dialog_supervisor_factory)
|
||||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt)
|
||||
recovery['validation_evidence'] = str(result.evidence_path or '')
|
||||
recovery['validation_passed'] = (result.status == 'PASS'
|
||||
and result.library_flag == 'GREEN' and result.safe_to_continue)
|
||||
if not recovery['validation_passed']:
|
||||
recovery['status'] = 'REPAIR_VALIDATION_FAILED'
|
||||
if result.status == 'PASS':
|
||||
result = replace(result, status='FAIL',
|
||||
stderr=(result.stderr+'\nREPAIR_VALIDATION_UNCONFIRMED: 未取得校验 GREEN').strip())
|
||||
else:
|
||||
recovery['status'] = 'REPAIRED'
|
||||
else:
|
||||
# A failed local database must not block unrelated CBBs once
|
||||
# every owner is released. Re-check target ownership here so
|
||||
# an incomplete repair record cannot authorize the queue.
|
||||
safe = False
|
||||
if (recovery.get('status') in {'MISSING_HEALTHY_BACKUP', 'NATIVE_REPAIR_FAILED'}
|
||||
and recovery.get('evidence', {}).get('failure_complete') is True
|
||||
and recovery.get('release', {}).get('safe_to_continue') is True):
|
||||
from .native_repair import select_repair_target
|
||||
try:
|
||||
target = select_repair_target(request, diagnosis)
|
||||
safe = (Path(recovery.get('project', '')).resolve() == target.project
|
||||
and Path(recovery.get('database', '')).resolve() == target.database)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
result = replace(result, safe_to_continue=safe, stderr=(result.stderr + '\n'
|
||||
+ str(recovery.get('status', 'REPAIR_UNCONFIRMED')) + ': '
|
||||
+ str(recovery.get('detail', '原生 Repair 未完成恢复'))).strip())
|
||||
result = replace(result, recovery_attempts=(recovery,))
|
||||
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
|
||||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||||
@@ -540,20 +653,57 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status,
|
||||
safe_to_continue=False,
|
||||
stderr=(result.stderr+f'\nREPAIR_UNCONFIRMED: {error}').strip())
|
||||
if result.evidence_path:
|
||||
try:
|
||||
result.evidence_path.write_text(json.dumps({'block_name': request.block_name, **asdict(result)},
|
||||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||||
except OSError:
|
||||
pass # Keep the native/repair errors even if evidence storage fails.
|
||||
release_confirmed = result.safe_to_continue
|
||||
if not release_confirmed and result.evidence_path:
|
||||
try:
|
||||
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
|
||||
release_confirmed = (release.get('worker_exited') is True
|
||||
and release.get('safe_to_continue') is True and release.get('block') == request.block_name
|
||||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve())
|
||||
if (release.get('worker_exited') is True and not release.get('error')
|
||||
and not release.get('close_errors') and not release.get('automation_release', {}).get('error')
|
||||
and release.get('block') == request.block_name
|
||||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
|
||||
remaining_clients = tuple(pid for pid in release.get('remaining_pids', [])
|
||||
if type(pid) is int and pid > 0)
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
return result
|
||||
finally:
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
try:
|
||||
if library_lock is not None:
|
||||
library_lock.__exit__(None, None, None)
|
||||
finally:
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
uninitialize_com(initialized)
|
||||
|
||||
|
||||
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
port_check=None, dialog_supervisor_factory=None):
|
||||
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
timing = RunTimings()
|
||||
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
|
||||
process_runner=process_runner, port_check=port_check,
|
||||
dialog_supervisor_factory=dialog_supervisor_factory)
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'} and result.evidence_path:
|
||||
dialog_supervisor_factory=dialog_supervisor_factory, version_prompt=version_prompt, stop=stop)
|
||||
if not request.auto_repair and result.evidence_path:
|
||||
try:
|
||||
release = json.loads(result.evidence_path.with_name('session-release.json').read_text('utf-8'))
|
||||
if (release.get('block') == request.block_name
|
||||
and Path(release['library']).resolve() == Path(request.lmc_file).resolve()):
|
||||
released = release.get('worker_exited') is True and release.get('safe_to_continue') is True
|
||||
result = replace(result, safe_to_continue=released)
|
||||
if not released and result.status == 'PASS':
|
||||
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
|
||||
job_path = result.evidence_path.with_name('request.json')
|
||||
if job_path.is_file():
|
||||
timing.begin('failure_diagnostics')
|
||||
@@ -579,8 +729,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
|
||||
detail = result.stderr
|
||||
if recovery and RECOVERY_ACTION not in detail:
|
||||
detail = (detail + '\n' + recovery_detail(recovery)).strip()
|
||||
result = replace(result, safe_to_continue=False, stderr=detail,
|
||||
error_category=ERROR_CODE)
|
||||
result = replace(result, stderr=detail, error_category=ERROR_CODE)
|
||||
if recovery and result.evidence_path:
|
||||
try:
|
||||
record_path = result.evidence_path.parent/'diagnostics/icdb-recovery.json'
|
||||
@@ -597,28 +746,29 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
|
||||
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
|
||||
if result.status not in {"PASS", "SKIPPED"} else ""))
|
||||
if result.evidence_path:
|
||||
result.evidence_path.write_text(json.dumps(
|
||||
{'block_name': request.block_name, **asdict(result)},
|
||||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||||
result.evidence_path.with_name('performance.json').write_text(json.dumps({
|
||||
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
|
||||
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
|
||||
'block_name': request.block_name, 'status': result.status,
|
||||
'unit': 'seconds', 'timings_seconds': durations,
|
||||
'incomplete_timing_phases': unfinished,
|
||||
'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)',
|
||||
'notes': 'phase totals overlap action/call subphases; skipped phases are absent',
|
||||
}, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
})
|
||||
if progress:
|
||||
progress({'stage': 'performance', 'block': request.block_name,
|
||||
_notify_progress(progress, {'stage': 'performance', 'block': request.block_name,
|
||||
'timings_seconds': durations, 'incomplete_timing_phases': unfinished,
|
||||
'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())})
|
||||
if request.logging_context:
|
||||
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
|
||||
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
|
||||
try:
|
||||
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
|
||||
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning('Operation logging failed', exc_info=True)
|
||||
return result
|
||||
|
||||
|
||||
def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||||
dialog_supervisor_factory=None):
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
"""Refresh one CBB and record its evidence.
|
||||
|
||||
``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``.
|
||||
@@ -627,12 +777,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
not depend on the host's port reservations.
|
||||
"""
|
||||
root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12])
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
try:
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
except OSError as error:
|
||||
return VerifyResult('FAIL', stderr=f'无法创建操作记录目录:{error}')
|
||||
fingerprint = runtime_fingerprint(request.mentor_bin)
|
||||
(root/'runtime.json').write_text(
|
||||
json.dumps(fingerprint, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
_write_optional_evidence(root/'runtime.json', fingerprint)
|
||||
if progress:
|
||||
progress({'stage': 'runtime_fingerprint', 'block': request.block_name,
|
||||
_notify_progress(progress, {'stage': 'runtime_fingerprint', 'block': request.block_name,
|
||||
**fingerprint})
|
||||
result_path = root/'result.json'
|
||||
runner = process_runner or (lambda command, timeout, cwd:
|
||||
@@ -650,6 +802,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
pass
|
||||
return replace(result, evidence_path=result_path)
|
||||
|
||||
if stop is not None and stop.is_set():
|
||||
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
|
||||
timing.begin('preflight')
|
||||
try:
|
||||
project, pcb = preflight(request)
|
||||
@@ -658,18 +812,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
finally:
|
||||
timing.end('preflight')
|
||||
|
||||
from .icdb_ownership import unreleased_addresses
|
||||
try:
|
||||
addresses = unreleased_addresses(request.lmc_file, project, pcb)
|
||||
if addresses:
|
||||
return finish(VerifyResult('BLOCKED', safe_to_continue=False,
|
||||
stderr='ICDB_SESSION_UNCONFIRMED: 原生数据库所有权未释放,未开始内容检查或校验:'
|
||||
+ '; '.join(addresses)))
|
||||
except OSError as error:
|
||||
return finish(VerifyResult('BLOCKED', safe_to_continue=False,
|
||||
stderr=f'ICDB_SESSION_UNCONFIRMED: 无法确认数据库所有权:{error}'))
|
||||
|
||||
content_check_enabled = bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
|
||||
content_check_enabled = not request.diagnostic_only and bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
|
||||
if content_check_enabled:
|
||||
try:
|
||||
executable = resolve_importer(request.mentor_bin)
|
||||
@@ -677,7 +820,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
return finish(VerifyResult('BLOCKED', stderr=str(error)))
|
||||
command = build_content_check_command(request, executable)
|
||||
if progress:
|
||||
progress({'stage': 'content_check_start', 'block': request.block_name,
|
||||
_notify_progress(progress, {'stage': 'content_check_start', 'block': request.block_name,
|
||||
'detail': ' '.join(command)})
|
||||
timing.begin('content_check')
|
||||
try:
|
||||
@@ -699,20 +842,20 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
stdout=stdout, report_path=report, content_check_exit_code=exit_code,
|
||||
safe_to_continue=not timed_out))
|
||||
if progress:
|
||||
progress({'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
|
||||
_notify_progress(progress, {'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
|
||||
|
||||
# The design server needs a port in 10000-10100; a system reservation there
|
||||
# blocks every item, so fail fast with the fix instead of timing out.
|
||||
# Sampling cannot establish that the whole native port range is unusable.
|
||||
from .mentor_adapter import icdb_ports_bindable
|
||||
check_ports = port_check or icdb_ports_bindable
|
||||
ports_ok, ports_detail = check_ports()
|
||||
if not ports_ok:
|
||||
if progress:
|
||||
progress({'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
|
||||
_notify_progress(progress, {'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
|
||||
if request.strict_port_check:
|
||||
return finish(VerifyResult('BLOCKED', stderr=ports_detail))
|
||||
|
||||
job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name,
|
||||
'diagnostic_only': request.diagnostic_only,
|
||||
'supervisor_pid': os.getpid(),
|
||||
'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds,
|
||||
'mentor_bin': str(request.mentor_bin or ''),
|
||||
@@ -723,11 +866,22 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
'desktop': 'CbbVerify_' + uuid4().hex[:12],
|
||||
'cancel_signal': str(root/'cancel-worker.signal'),
|
||||
'start_gate': str(root/'start-worker.signal')}
|
||||
consent = None
|
||||
job_path = root/'request.json'
|
||||
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
|
||||
try:
|
||||
if request.post_repair_proof is not None:
|
||||
from .post_repair_consent import PostRepairProof, ConsentController, exact_prompt
|
||||
if not isinstance(request.post_repair_proof, PostRepairProof):
|
||||
return finish(VerifyResult('BLOCKED', stderr='POST_REPAIR_PROOF_REQUIRED'))
|
||||
job['post_repair_consent'] = request.post_repair_proof.for_job(job)
|
||||
consent = ConsentController(request.post_repair_proof, job)
|
||||
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
|
||||
except Exception as error:
|
||||
return finish(VerifyResult('FAIL', stderr=str(error)))
|
||||
events = root/'events.jsonl'
|
||||
offset = 0
|
||||
target_pid = None
|
||||
launcher_pid = None
|
||||
lm_pid = None
|
||||
verify_invoked = False
|
||||
invoke_returned = False
|
||||
@@ -743,10 +897,13 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
lock_status = win32event.WaitForSingleObject(mutex, 0)
|
||||
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
|
||||
mutex.Close()
|
||||
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。', safe_to_continue=False))
|
||||
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。',
|
||||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY'))
|
||||
worker_launch_attempted = False
|
||||
try:
|
||||
timing.begin('worker_setup')
|
||||
with (root/'worker.log').open('w', encoding='utf-8') as log:
|
||||
worker_launch_attempted = True
|
||||
if request.interactive_input:
|
||||
process = DesktopWorker(
|
||||
worker_command(job_path), None, interactive_input=True)
|
||||
@@ -757,15 +914,17 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
deadline = ActivityDeadline(request.timeout_seconds)
|
||||
last_activity_key = None
|
||||
phase = DialogPhase.PCB_STARTING
|
||||
restore_controller = None
|
||||
dialog_agent = None
|
||||
latest_snapshots = ()
|
||||
if dialog_supervisor_factory is None:
|
||||
if hasattr(process, 'start_companion'):
|
||||
from .dialog_agent import DialogAgentClient
|
||||
from .dialog_rules import default_rules_path, load_rules
|
||||
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent')
|
||||
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent', stop=stop)
|
||||
dialog_supervisor = DialogSupervisor(
|
||||
AgentDialogBackend(dialog_agent),
|
||||
AgentDialogBackend(dialog_agent, pids=lambda: {
|
||||
int(pid) for pid in (target_pid, lm_pid, launcher_pid) if pid}),
|
||||
rules=load_rules(default_rules_path()),
|
||||
)
|
||||
else:
|
||||
@@ -778,8 +937,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
desktop=process.desktop,
|
||||
pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
|
||||
)
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
|
||||
Path(job['start_gate']).touch()
|
||||
while process.poll() is None:
|
||||
if stop is not None and stop.is_set():
|
||||
_stop_worker(process)
|
||||
return finish(VerifyResult('CANCELLED', stderr='已取消当前校验。',
|
||||
safe_to_continue=False, error_stage=phase.value))
|
||||
if events.exists():
|
||||
with events.open(encoding='utf-8') as stream:
|
||||
stream.seek(offset)
|
||||
@@ -790,6 +955,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
break
|
||||
timing.record(event)
|
||||
if event.get('pcb_pid'): target_pid = event['pcb_pid']
|
||||
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
|
||||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||||
stage = event.get('stage')
|
||||
if stage == 'verify_invoke': verify_invoked = True
|
||||
@@ -813,7 +979,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
0.1, float(request.timeout_seconds))
|
||||
deadline.touch()
|
||||
last_activity_key = activity_key
|
||||
if progress: progress(event)
|
||||
if progress: _notify_progress(progress, event)
|
||||
offset = stream.tell()
|
||||
if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5:
|
||||
from .post_verify_diagnostics import observe_post_verify
|
||||
@@ -841,6 +1007,27 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
dialog_result = dialog_supervisor.poll()
|
||||
else:
|
||||
dialog_result = dialog_supervisor.poll()
|
||||
if (consent is not None and dialog_agent is not None
|
||||
and dialog_result.error_code == 'DATABASE_RECOVERY_REQUIRED'
|
||||
and dialog_result.snapshot is not None and exact_prompt(dialog_result.snapshot)):
|
||||
consent_snapshot = dialog_result.snapshot
|
||||
if consent_snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}:
|
||||
dialog_result = consent.handle(consent_snapshot, dialog_result.observed_snapshots,
|
||||
target_pid=target_pid, launcher_pid=launcher_pid, worker_pid=process.pid,
|
||||
companion_pids=tuple(process.companion_pids), perform=dialog_supervisor.backend.perform)
|
||||
if dialog_result.error_code == 'PCB_VERSION_CHOICE_REQUIRED' and dialog_result.snapshot is not None:
|
||||
from .dialog_rules import archive_version_choice
|
||||
choice_path = archive_version_choice(dialog_result.snapshot, root)
|
||||
dialog_result = replace(dialog_result, detail=dialog_result.detail + f'\n版本候选与原生窗口证据:{choice_path}')
|
||||
if (version_prompt is not None and consent is not None and dialog_agent is not None
|
||||
and dialog_result.snapshot.hwnd in {int(h) for h in process.desktop.EnumDesktopWindows()}):
|
||||
from .pcb_restore_prompt import RestoreController
|
||||
if restore_controller is None:
|
||||
restore_controller = RestoreController(job)
|
||||
dialog_result = restore_controller.handle(dialog_result.snapshot,
|
||||
dialog_result.observed_snapshots, target_pid=target_pid, launcher_pid=launcher_pid,
|
||||
worker_pid=process.pid, companion_pids=tuple(process.companion_pids),
|
||||
show=version_prompt, perform=dialog_supervisor.backend.perform)
|
||||
# Normal progress stays in memory. Only actions/errors and the
|
||||
# last failure snapshots are persisted, not every elapsed tick.
|
||||
latest_snapshots = dialog_result.observed_snapshots
|
||||
@@ -866,7 +1053,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
if dialog_result.error_code:
|
||||
_write_failure_windows(root, latest_snapshots)
|
||||
_write_process_tree(root, process, target_pid)
|
||||
_stop_worker(process)
|
||||
# A failed database load has no document to close. Give the
|
||||
# worker time to normally close its owned empty PCB before
|
||||
# recovery checks for remaining native database clients.
|
||||
_stop_worker(process, timeout=40)
|
||||
return finish(VerifyResult(
|
||||
'FAIL',
|
||||
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
|
||||
@@ -893,14 +1083,14 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
if dispatch_failure is not None:
|
||||
status, detail = dispatch_failure
|
||||
if progress:
|
||||
progress({
|
||||
_notify_progress(progress, {
|
||||
'stage': 'verify_dispatch_failed',
|
||||
'block': request.block_name,
|
||||
'detail': detail,
|
||||
'observation': last_post_verify_observation,
|
||||
})
|
||||
return finish(VerifyResult(status, stderr=detail, safe_to_continue=False))
|
||||
return finish(VerifyResult('TIMEOUT', stderr='操作超时;已停止自动化并保留原生会话,停止批次,请检查恢复记录。', error_stage=phase.value, safe_to_continue=False))
|
||||
return finish(VerifyResult('TIMEOUT', stderr='操作超时,当前项未完成。', error_stage=phase.value, safe_to_continue=False))
|
||||
time.sleep(.2)
|
||||
worker_result = root/'outcome.json'
|
||||
if not worker_result.exists():
|
||||
@@ -912,9 +1102,25 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
except Exception as error:
|
||||
if 'process' in locals() and process.poll() is None:
|
||||
_stop_worker(process)
|
||||
return finish(VerifyResult('FAIL', stderr=str(error), safe_to_continue=False))
|
||||
launch_release = getattr(error, 'worker_launch_release', None) if 'process' not in locals() else None
|
||||
if not worker_launch_attempted:
|
||||
launch_release = {'process_created': False, 'worker_exited': True,
|
||||
'safe_to_continue': True, 'remaining_pids': []}
|
||||
no_launch = (isinstance(launch_release, dict)
|
||||
and launch_release.get('process_created') is False
|
||||
and launch_release.get('worker_exited') is True
|
||||
and launch_release.get('safe_to_continue') is True
|
||||
and launch_release.get('remaining_pids') == [])
|
||||
if no_launch:
|
||||
_write_optional_evidence(root/'session-release.json', {
|
||||
**launch_release, 'block': request.block_name, 'library': str(request.lmc_file),
|
||||
'worker_pid': None})
|
||||
status = 'CANCELLED' if stop is not None and stop.is_set() else 'FAIL'
|
||||
return finish(VerifyResult(status, stderr=str(error), safe_to_continue=no_launch))
|
||||
finally:
|
||||
close_errors = []
|
||||
worker_exited = False
|
||||
automation_release = {'safe_to_continue': False}
|
||||
if 'dialog_agent' in locals() and dialog_agent is not None:
|
||||
try:
|
||||
dialog_agent.close()
|
||||
@@ -922,15 +1128,37 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
close_errors.append(str(error))
|
||||
if 'process' in locals():
|
||||
try:
|
||||
worker_exited = process.poll() is not None
|
||||
if not worker_exited:
|
||||
worker_exited = _stop_worker(process)
|
||||
if hasattr(process, 'automation_release_state'):
|
||||
automation_release = process.automation_release_state()
|
||||
else:
|
||||
automation_release = {'safe_to_continue': worker_exited, 'remaining_pids': []}
|
||||
process.close()
|
||||
except Exception as error:
|
||||
close_errors.append(str(error))
|
||||
if 'process' in locals():
|
||||
try:
|
||||
if events.is_file():
|
||||
with events.open(encoding='utf-8') as stream:
|
||||
stream.seek(offset)
|
||||
for line in stream:
|
||||
try:
|
||||
event = json.loads(line)
|
||||
timing.record(event)
|
||||
if event.get('pcb_pid'): target_pid = event['pcb_pid']
|
||||
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
|
||||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||||
if event.get('stage') == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
offset = stream.tell()
|
||||
release_result = _cleanup_session_processes(
|
||||
root,
|
||||
lm_pid=lm_pid,
|
||||
target_pid=target_pid,
|
||||
launcher_pid=launcher_pid,
|
||||
pcb=pcb,
|
||||
pcb_baseline=pcb_baseline,
|
||||
progress=progress,
|
||||
@@ -941,16 +1169,28 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
from .icdb_ownership import unreleased_addresses
|
||||
addresses = unreleased_addresses(request.lmc_file, project, pcb)
|
||||
if addresses:
|
||||
release_result = {**(release_result or {}), 'safe_to_continue': False,
|
||||
'icdb_address': addresses[0], 'icdb_addresses': addresses}
|
||||
release_result = {**(release_result or {}), 'icdb_addresses': addresses}
|
||||
except OSError as error:
|
||||
close_errors.append(str(error))
|
||||
if close_errors:
|
||||
release_result = {**(release_result or {}), 'safe_to_continue': False,
|
||||
'close_errors': close_errors}
|
||||
if not worker_exited or not automation_release.get('safe_to_continue'):
|
||||
release_result = {**(release_result or {}), 'safe_to_continue': False,
|
||||
'automation_release': automation_release, 'worker_exited': worker_exited}
|
||||
release_result['remaining_pids'] = sorted(set(release_result.get('remaining_pids', []))
|
||||
| set(automation_release.get('remaining_pids', [])))
|
||||
try:
|
||||
(root/'session-release.json').write_text(json.dumps({
|
||||
**(release_result or {}), 'block': request.block_name,
|
||||
'library': str(request.lmc_file), 'worker_exited': worker_exited,
|
||||
'worker_pid': getattr(process, 'pid', None)},
|
||||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||||
except OSError:
|
||||
pass
|
||||
if not release_result or not release_result.get('safe_to_continue'):
|
||||
if 'result' in locals():
|
||||
result = replace(result, status='FAIL', safe_to_continue=False,
|
||||
result = replace(result, status='FAIL' if result.status == 'PASS' else result.status, safe_to_continue=False,
|
||||
stderr=(result.stderr + '\nPCB_RELEASE_UNCONFIRMED: 未确认原生会话退出').strip())
|
||||
recovery = {'desktop': job['desktop'], 'project': str(project),
|
||||
'pcb': str(pcb), 'release': release_result,
|
||||
@@ -978,22 +1218,22 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
|
||||
if result.status == 'PASS':
|
||||
if result.status == 'PASS' and not request.diagnostic_only:
|
||||
timing.begin('library_flag')
|
||||
bridge_records = []
|
||||
def record_attempt(record):
|
||||
bridge_records.append(record)
|
||||
if progress:
|
||||
progress({'stage': 'library_flag_probe', 'block': request.block_name, **record})
|
||||
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
|
||||
try:
|
||||
probe = read_library_flag_with_retry(
|
||||
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
|
||||
request.block_name, on_attempt=record_attempt)
|
||||
result = apply_post_verify_status(result, probe)
|
||||
result = replace(result, bridge_attempts=tuple(bridge_records))
|
||||
(root/'bridge-status.json').write_text(json.dumps(bridge_records, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
_write_optional_evidence(root/'bridge-status.json', bridge_records)
|
||||
if progress:
|
||||
progress({'stage': 'library_flag', 'block': request.block_name,
|
||||
_notify_progress(progress, {'stage': 'library_flag', 'block': request.block_name,
|
||||
'library_flag': result.library_flag, 'detail': result.library_flag_detail})
|
||||
finally:
|
||||
timing.end('library_flag')
|
||||
@@ -1019,10 +1259,10 @@ def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=Non
|
||||
result = VerifyResult('FAIL', stderr=str(error), safe_to_continue=False)
|
||||
results.append(result)
|
||||
# Never start another writer after an unconfirmed native shutdown.
|
||||
queue_stop = queue_stop or result.status == 'QUEUE_STOP' or not result.safe_to_continue
|
||||
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'} or not result.safe_to_continue
|
||||
if queue_stop and stop is not None:
|
||||
stop.set()
|
||||
if progress: progress(request.block_name, result)
|
||||
if progress: _notify_progress(progress, request.block_name, result)
|
||||
return results
|
||||
|
||||
|
||||
@@ -1043,11 +1283,21 @@ def write_failure_report(requests, results, root, *, path=None):
|
||||
root = Path(root); root.mkdir(parents=True, exist_ok=True)
|
||||
path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||||
failed = [(req, result) for req, result in zip(requests, results)
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}]
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}
|
||||
and not (result.status in {'CHECKED', 'REPAIRED'} and result.safe_to_continue)]
|
||||
passed = sum(result.status == 'PASS' for result in results)
|
||||
skipped = len(results)-passed-len(failed)
|
||||
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped}',
|
||||
checked = sum(result.status == 'CHECKED' and result.safe_to_continue for result in results)
|
||||
repaired = sum(result.status == 'REPAIRED' and result.safe_to_continue for result in results)
|
||||
skipped = len(results)-passed-len(failed)-checked-repaired
|
||||
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped} 仅检查未修复: {checked} ICDB已修复待校验: {repaired}',
|
||||
'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)']
|
||||
if requests and all(req.maintenance for req in requests):
|
||||
from .icdb_maintenance import maintenance_counts
|
||||
counts = maintenance_counts(results)
|
||||
lines[:2] = ['ICDB 检查异常清单',
|
||||
f'总数: {len(results)} 未发现异常: {checked} 发现问题: {counts["issues_found"]} '
|
||||
f'占用: {counts["in_use"]} 无法确认: {counts["unconfirmed"]} '
|
||||
f'取消: {counts["cancelled"]} 未执行: {counts["skipped"]}']
|
||||
for req, result in failed:
|
||||
timings = ' '.join(f'{key}={value:.3f}s' +
|
||||
('(unfinished)' if key in result.incomplete_timing_phases else '')
|
||||
@@ -1068,6 +1318,8 @@ def write_batch_report(requests, results, root):
|
||||
path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||||
lines = ['CBB Verify 批量结果',
|
||||
'流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)']
|
||||
if requests and all(req.maintenance for req in requests):
|
||||
lines = ['ICDB 批量检查结果', '流程: 原生 PCB 打开检查;不执行 Repair、Restore 或 CBB 校验。']
|
||||
for req, result in zip(requests, results):
|
||||
detail = _report_field(result.stderr or result.stdout.strip()[:200])
|
||||
seconds = ' '.join(f'{key}={value:.3f}s' +
|
||||
|
||||
Reference in new issue
Block a user