fix: close owned PCB startup failures before ICDB recovery

This commit is contained in:
阿白叔 committed 2026-11-02 17:40:18 +08:00
1 parent 39985f7cfd
commit ec78dd722c
5 files changed
+360 -1

No files matched your search

@@ -0,0 +1,80 @@
# Issue #2: release the PCB launched by a failed database open
Branch: `codex/temp-issue-2-icdb-recovery`.
## Field failure
The issue attachment records an iCDB lost-server-session error while opening
`1GPHY_8211`. The supervisor declined the database log viewer and cancelled the
worker. Because document identity had not been established, the existing
document-close helper was never called. PCB and its launcher remained alive;
the subsequent Repair attempt returned `REPAIR_DATABASE_IN_USE` and the other
380 queued entries were skipped. There is no evidence of native Repair actually
running on that field database.
## Change
The worker now has a separate startup-failure close path. It requires a current
iCDB diagnosis or a database-load-failure dialog belonging to this PCB during
startup. It checks the private desktop, excludes pre-existing process IDs, and
matches the complete PCB path and ReusableCircuit argument. Only an enabled
empty PCB main window, without modal dialogs, MDI documents or database address
records, receives a cooperative `WM_CLOSE`. A requested close is not treated as
success until process exit is observed. Ambiguous or busy sessions are retained.
The supervisor allows up to 40 seconds for fatal-error cleanup so that the
worker's 20-second startup-close window can complete before recovery begins.
The existing repair preconditions, backups and success checks remain in force.
Each attempt writes `startup-close.json` and close-request/result events.
## Verification
Regression tests cover the missing pre-identity cleanup route, real observed
empty-window titles, exact path matching, pre-existing processes, another
desktop, live address records, modal and MDI windows, disabled windows, unrelated
failure records and a close request that does not result in exit.
Delayed dialog/address release and delayed process exit are also covered.
Validation: `python -m pytest -q` — 491 passed, one existing pywinauto COM-mode
warning; `python -m compileall -q src tests` and `git diff --check` passed.
A new isolated local library was created with the existing
`tools/validate_icdb_open_repair_reopen.py` tool, output
`diagnostics/issue2-startup-20261102`. No production library was changed.
The test damages only the new copy's database header; it does not reproduce the
field server-session-loss condition.
The real Mentor opening run produced `MENTOR_DATABASE_LOAD_FAILED`, then:
- `startup_close_requested` for the launched PCB PID 6532;
- `startup_close_result: CLOSED`;
- worker and supervisor both recorded `process_exit_confirmed`, with no
remaining PCB processes.
The subsequent separate native Repair run was able to start and completed as
`RESTORED_REVIEW_REQUIRED`, using the fixture's historical backup. This proves
the failed-open session no longer blocks the next repair operation in this
fixture. It does not prove automatic recovery of the field database or establish
that an older backup contains the desired design revision.
The separate `open-after` run on the same restored fixture completed native
integration, Save/IsSaved/Close/Quit, confirmed process release and returned
`PASS`, `safe_to_continue=True`, `READY (IsVerifiedReuseBlock, raw=1)`.
The `compare` phase passed all seven checks, including absence of the original
opening error and unchanged source-fixture database SHA-256. Its
`server_session_loss_reproduced` field is explicitly `false`.
Reproduction commands (set `PYTHONPATH=src`; setup requires a new output name):
```powershell
python tools/validate_icdb_open_repair_reopen.py setup --output issue2-startup-20261102
python tools/validate_icdb_open_repair_reopen.py open-before --output issue2-startup-20261102
python tools/validate_icdb_open_repair_reopen.py repair --output issue2-startup-20261102
python tools/validate_icdb_open_repair_reopen.py open-after --output issue2-startup-20261102
python tools/validate_icdb_open_repair_reopen.py compare --output issue2-startup-20261102
```
The stages are deliberately separate: the test fixture has a format-corruption
error and historical-backup recovery, not the field lost-session auto-repair
trigger. The test does not change production handling of old-backup recovery.
READY filtering, hash-scope changes and CBB CRUD are outside this focused fix.
+110
View File
@@ -5,6 +5,116 @@ from pathlib import Path
import time
def _command_args(command):
import ctypes
from ctypes import wintypes
shell = ctypes.WinDLL('shell32', use_last_error=True)
kernel = ctypes.WinDLL('kernel32', use_last_error=True)
shell.CommandLineToArgvW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(ctypes.c_int)]
shell.CommandLineToArgvW.restype = ctypes.POINTER(wintypes.LPWSTR)
kernel.LocalFree.argtypes = [wintypes.HLOCAL]
kernel.LocalFree.restype = wintypes.HLOCAL
count = ctypes.c_int()
args = shell.CommandLineToArgvW(command, ctypes.byref(count))
if not args:
raise ctypes.WinError(ctypes.get_last_error())
try:
return list(args[:count.value])
finally:
kernel.LocalFree(ctypes.cast(args, wintypes.HLOCAL))
def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: None,
timeout_seconds=20):
"""Normally close our empty PCB after a diagnosed database startup failure.
No COM attach: there is no document, and the global ROT can name another
session. Unknown windows, live database addresses and other desktops are
retained. WM_CLOSE is cooperative; no save prompt is accepted or killed.
"""
import win32con, win32gui, win32process
from .mentor_adapter import _process_details
from .native_repair_ui import _current_desktop_name
from .icdb_ownership import unreleased_addresses
root = Path(job['root'])
result = {'status': 'RETAINED', 'pcb_pid': pcb_pid}
try:
diagnosed = False
diagnosis_path = root/'diagnostics/icdb-recovery.json'
if diagnosis_path.is_file():
diagnosis = json.loads(diagnosis_path.read_text('utf-8'))
diagnosed = diagnosis.get('error_code') == 'ICDB_REPAIR_REQUIRED'
dialogs_path = root/'dialogs.jsonl'
if not diagnosed and dialogs_path.is_file():
for line in dialogs_path.read_text('utf-8').splitlines():
try:
dialog = json.loads(line)
except ValueError:
continue
if (dialog.get('pid') == pcb_pid and dialog.get('phase') == 'PCB_STARTING'
and dialog.get('error_code') == 'MENTOR_DATABASE_LOAD_FAILED'):
diagnosed = True
if not diagnosed:
raise ValueError('STARTUP_FAILURE_UNCONFIRMED')
desktop = job.get('desktop', '')
if (job.get('interactive_input') or not desktop.startswith('CbbVerify_')
or _current_desktop_name() != desktop):
raise ValueError('STARTUP_DESKTOP_UNCONFIRMED')
if not pcb_pid or pcb_pid in baseline_pids:
raise ValueError('STARTUP_PROCESS_UNCONFIRMED')
expected = os.path.normcase(os.path.abspath(job['pcb']))
deadline = time.monotonic() + max(0, timeout_seconds)
sent = False
while True:
records = [r for r in _process_details('ExpeditionPCB.exe') if r[0] == pcb_pid]
if not records:
result['status'] = 'CLOSED'
break
_, executable, command, _ = records[0]
args = _command_args(command)
if (Path(executable).name.casefold() != 'expeditionpcb.exe'
or '-reusablecircuit' not in [a.casefold() for a in args]
or expected not in [os.path.normcase(os.path.abspath(a)) for a in args[1:]]):
raise ValueError('STARTUP_PROCESS_UNCONFIRMED')
if not sent:
windows = []
def observe(hwnd, _):
if (win32process.GetWindowThreadProcessId(hwnd)[1] == pcb_pid
and win32gui.IsWindowVisible(hwnd)):
windows.append(hwnd)
win32gui.EnumWindows(observe, None)
main = [w for w in windows if win32gui.GetClassName(w).startswith('Afx:')
and win32gui.GetWindowText(w).strip() in (
'Expedition PCB', 'Expedition PCB - Reusable Block')
and win32gui.IsWindowEnabled(w)]
dialogs = any(win32gui.GetClassName(w) == '#32770' for w in windows)
documents = []
def child(hwnd, _):
if (win32gui.GetClassName(hwnd) == 'MDIClient'
and win32gui.GetWindow(hwnd, win32con.GW_CHILD)):
documents.append(hwnd)
if len(main) == 1:
win32gui.EnumChildWindows(main[0], child, None)
addresses = unreleased_addresses(job['lmc_file'], job['project'], job['pcb'])
if len(main) == 1 and not dialogs and not documents and not addresses:
win32gui.PostMessage(main[0], win32con.WM_CLOSE, 0, 0)
sent = True
emit(stage='startup_close_requested', pcb_pid=pcb_pid, hwnd=main[0])
# Observe exit even when the caller requested no extra wait.
continue
result['detail'] = 'STARTUP_IDLE_UNCONFIRMED: 等待空闲窗口和数据库释放'
if time.monotonic() >= deadline:
if sent:
result['detail'] = 'STARTUP_EXIT_UNCONFIRMED: 已请求正常关闭,但 PCB 尚未退出'
break
time.sleep(.2)
except Exception as error:
result['detail'] = str(error)
(root/'startup-close.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
emit(stage='startup_close_result', **result)
return result
def close_failed_document(job, *, emit=lambda **event: None, timeout_seconds=30):
from .desktop_worker import launch_save_helper
root = Path(job['root'])
+8
View File
@@ -863,6 +863,14 @@ def execute_job(job, emit):
lm_session.close()
except Exception as error:
emit(stage='failure_close_unconfirmed', detail=str(error))
elif not completed and worker_pid is not None:
try:
from .failed_session import close_failed_startup
failed_close = close_failed_startup(job, worker_pid, baseline_pids=pcb_before, emit=emit)
if failed_close.get('status') == 'CLOSED' and lm_session is not None:
lm_session.close()
except Exception as error:
emit(stage='startup_close_unconfirmed', detail=str(error))
cleanup_verify_processes(
lm_pid,
worker_pid,
+4 -1
View File
@@ -866,7 +866,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
if dialog_result.error_code:
_write_failure_windows(root, latest_snapshots)
_write_process_tree(root, process, target_pid)
_stop_worker(process)
# A failed database load has no document to close. Give the
# worker time to normally close its owned empty PCB before
# recovery checks for remaining native database clients.
_stop_worker(process, timeout=40)
return finish(VerifyResult(
'FAIL',
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
+158
View File
@@ -0,0 +1,158 @@
import json
from pathlib import Path
import pytest
from cbb_refresh_verify import failed_session, mentor_adapter
def startup_fixture(tmp_path, monkeypatch):
import win32gui, win32process
from cbb_refresh_verify import native_repair_ui
job = {'root': str(tmp_path), 'pcb': str(tmp_path/'PCB/CBB.pcb'),
'project': str(tmp_path/'CBB.prj'), 'lmc_file': str(tmp_path/'lib.lmc'),
'desktop': 'CbbVerify_test'}
diagnostics = tmp_path/'diagnostics'
diagnostics.mkdir()
(diagnostics/'icdb-recovery.json').write_text(json.dumps({
'error_code': 'ICDB_REPAIR_REQUIRED', 'scope': 'project',
'database_path': str(tmp_path/'database'), 'session': 'lost-session'}))
records = [(12, r'C:\Mentor\wg\ExpeditionPCB.exe',
f'ExpeditionPCB.exe "{job["pcb"]}" -ReusableCircuit', 11)]
monkeypatch.setattr(mentor_adapter, '_process_details', lambda _: list(records))
monkeypatch.setattr(native_repair_ui, '_current_desktop_name', lambda: 'CbbVerify_test')
monkeypatch.setattr(win32gui, 'EnumWindows', lambda fn, arg: fn(100, arg))
monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda *args: None)
monkeypatch.setattr(win32process, 'GetWindowThreadProcessId', lambda _: (1, 12))
monkeypatch.setattr(win32gui, 'IsWindowVisible', lambda _: True)
monkeypatch.setattr(win32gui, 'IsWindowEnabled', lambda _: True)
monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: 'Expedition PCB')
monkeypatch.setattr(win32gui, 'GetClassName', lambda _: 'Afx:main')
messages = []
def post(hwnd, message, wparam, lparam):
messages.append((hwnd, message))
records.clear()
monkeypatch.setattr(win32gui, 'PostMessage', post)
return job, records, messages
@pytest.mark.parametrize('title', ['Expedition PCB', 'Expedition PCB - Reusable Block'])
def test_failed_startup_normally_closes_owned_empty_pcb(tmp_path, monkeypatch, title):
import win32con
import win32gui
job, records, messages = startup_fixture(tmp_path, monkeypatch)
monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: title)
result = failed_session.close_failed_startup(job, 12, baseline_pids=(), timeout_seconds=0)
assert result['status'] == 'CLOSED'
assert not records
assert messages == [(100, win32con.WM_CLOSE)]
assert json.loads((tmp_path/'startup-close.json').read_text())['status'] == 'CLOSED'
@pytest.mark.parametrize('obstacle', ['baseline', 'wrong_path', 'path_prefix', 'desktop', 'address',
'dialog', 'document_title', 'mdi_document', 'disabled', 'no_diagnosis'])
def test_failed_startup_preserves_unconfirmed_or_busy_session(tmp_path, monkeypatch, obstacle):
import win32gui
from cbb_refresh_verify import native_repair_ui
job, records, messages = startup_fixture(tmp_path, monkeypatch)
baseline = ()
if obstacle == 'baseline': baseline = (12,)
if obstacle == 'wrong_path': records[0] = (12, records[0][1], 'other.pcb -ReusableCircuit', 11)
if obstacle == 'path_prefix':
records[0] = (12, records[0][1], f'pcb.exe "{job["pcb"]}.other" -ReusableCircuit', 11)
if obstacle == 'desktop': monkeypatch.setattr(native_repair_ui, '_current_desktop_name', lambda: 'Default')
if obstacle == 'address':
address = tmp_path/'database/cdbsvr/sAddress.adr'
address.parent.mkdir(parents=True)
address.write_text('owner')
if obstacle == 'dialog': monkeypatch.setattr(win32gui, 'GetClassName', lambda _: '#32770')
if obstacle == 'document_title': monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: 'CBB.pcb - Expedition PCB')
if obstacle == 'mdi_document':
monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda hwnd, fn, arg: fn(101, arg))
monkeypatch.setattr(win32gui, 'GetClassName', lambda hwnd: 'MDIClient' if hwnd == 101 else 'Afx:main')
monkeypatch.setattr(win32gui, 'GetWindow', lambda *args: 102)
if obstacle == 'disabled': monkeypatch.setattr(win32gui, 'IsWindowEnabled', lambda _: False)
if obstacle == 'no_diagnosis': (tmp_path/'diagnostics/icdb-recovery.json').unlink()
result = failed_session.close_failed_startup(job, 12, baseline_pids=baseline, timeout_seconds=0)
assert result['status'] == 'RETAINED'
assert records
assert not messages
def test_requested_close_without_exit_is_not_reported_closed(tmp_path, monkeypatch):
import win32gui
job, records, messages = startup_fixture(tmp_path, monkeypatch)
monkeypatch.setattr(win32gui, 'PostMessage', lambda hwnd, message, *_: messages.append((hwnd, message)))
result = failed_session.close_failed_startup(job, 12, timeout_seconds=0)
assert result['status'] == 'RETAINED'
assert 'STARTUP_EXIT_UNCONFIRMED' in result['detail']
assert len(messages) == 1
assert records
@pytest.mark.parametrize('obstacle', ['address', 'dialog', 'process'])
def test_startup_close_waits_for_native_state_transitions(tmp_path, monkeypatch, obstacle):
from types import SimpleNamespace
import win32gui
job, records, messages = startup_fixture(tmp_path, monkeypatch)
ticks = [0]
address = tmp_path/'database/cdbsvr/sAddress.adr'
if obstacle == 'address':
address.parent.mkdir(parents=True)
address.write_text('owner')
if obstacle == 'dialog':
monkeypatch.setattr(win32gui, 'GetClassName',
lambda _: '#32770' if ticks[0] == 0 else 'Afx:main')
if obstacle == 'process':
monkeypatch.setattr(win32gui, 'PostMessage',
lambda hwnd, message, *_: messages.append((hwnd, message)))
def advance(seconds):
ticks[0] += seconds
if obstacle == 'address' and address.exists(): address.unlink()
if obstacle == 'process' and messages: records.clear()
monkeypatch.setattr(failed_session, 'time', SimpleNamespace(
monotonic=lambda: ticks[0], sleep=advance))
result = failed_session.close_failed_startup(job, 12, timeout_seconds=1)
assert ticks[0] > 0
assert result['status'] == 'CLOSED'
assert len(messages) == 1
assert not records
def test_worker_closes_startup_failure_before_document_identity(tmp_path, monkeypatch):
from test_verify_worker_stability import exercise_worker
job, events, execute = exercise_worker(tmp_path, monkeypatch)
job['desktop'] = 'CbbVerify_test'
def fail_open(*args, **kwargs):
raise RuntimeError('VERIFY_CANCELLED: database open failed')
monkeypatch.setattr('cbb_refresh_verify.pcb_bridge.wait_for_target_document', fail_open)
# Native process/window behavior is covered above; here verify the worker
# actually reaches startup cleanup instead of requiring a loaded document.
def close(job, pid, **kwargs):
kwargs['emit'](stage='startup_close_result', status='CLOSED', pcb_pid=pid)
return {'status': 'CLOSED'}
monkeypatch.setattr(failed_session, 'close_failed_startup', close, raising=False)
with pytest.raises(RuntimeError, match='VERIFY_CANCELLED'):
execute()
assert any(e['stage'] == 'startup_close_result' and e['pcb_pid'] == 2 for e in events)
assert not any(e['stage'] == 'save_start' for e in events)
def test_failed_startup_accepts_current_owned_database_failure_dialog(tmp_path, monkeypatch):
job, records, messages = startup_fixture(tmp_path, monkeypatch)
(tmp_path/'diagnostics/icdb-recovery.json').unlink()
(tmp_path/'dialogs.jsonl').write_text(json.dumps({
'pid': 12, 'phase': 'PCB_STARTING', 'error_code': 'MENTOR_DATABASE_LOAD_FAILED'})+'\n')
result = failed_session.close_failed_startup(job, 12, timeout_seconds=0)
assert result['status'] == 'CLOSED'
assert not records
@pytest.mark.parametrize('pid,phase', [(99, 'PCB_STARTING'), (12, 'SAVING')])
def test_failed_startup_rejects_unrelated_failure_dialog(tmp_path, monkeypatch, pid, phase):
job, records, messages = startup_fixture(tmp_path, monkeypatch)
(tmp_path/'diagnostics/icdb-recovery.json').unlink()
(tmp_path/'dialogs.jsonl').write_text(json.dumps({
'pid': pid, 'phase': phase, 'error_code': 'MENTOR_DATABASE_LOAD_FAILED'})+'\n')
assert failed_session.close_failed_startup(job, 12, timeout_seconds=0)['status'] == 'RETAINED'
assert not messages