diff --git a/docs/releases/2026-11-02-issue-2-startup-recovery.md b/docs/releases/2026-11-02-issue-2-startup-recovery.md new file mode 100644 index 0000000..38f43d6 --- /dev/null +++ b/docs/releases/2026-11-02-issue-2-startup-recovery.md @@ -0,0 +1,80 @@ +# Issue #2: release the PCB launched by a failed database open + +Branch: `codex/temp-issue-2-icdb-recovery`. + +## Field failure + +The issue attachment records an iCDB lost-server-session error while opening +`1GPHY_8211`. The supervisor declined the database log viewer and cancelled the +worker. Because document identity had not been established, the existing +document-close helper was never called. PCB and its launcher remained alive; +the subsequent Repair attempt returned `REPAIR_DATABASE_IN_USE` and the other +380 queued entries were skipped. There is no evidence of native Repair actually +running on that field database. + +## Change + +The worker now has a separate startup-failure close path. It requires a current +iCDB diagnosis or a database-load-failure dialog belonging to this PCB during +startup. It checks the private desktop, excludes pre-existing process IDs, and +matches the complete PCB path and ReusableCircuit argument. Only an enabled +empty PCB main window, without modal dialogs, MDI documents or database address +records, receives a cooperative `WM_CLOSE`. A requested close is not treated as +success until process exit is observed. Ambiguous or busy sessions are retained. + +The supervisor allows up to 40 seconds for fatal-error cleanup so that the +worker's 20-second startup-close window can complete before recovery begins. +The existing repair preconditions, backups and success checks remain in force. +Each attempt writes `startup-close.json` and close-request/result events. + +## Verification + +Regression tests cover the missing pre-identity cleanup route, real observed +empty-window titles, exact path matching, pre-existing processes, another +desktop, live address records, modal and MDI windows, disabled windows, unrelated +failure records and a close request that does not result in exit. +Delayed dialog/address release and delayed process exit are also covered. +Validation: `python -m pytest -q` — 491 passed, one existing pywinauto COM-mode +warning; `python -m compileall -q src tests` and `git diff --check` passed. + +A new isolated local library was created with the existing +`tools/validate_icdb_open_repair_reopen.py` tool, output +`diagnostics/issue2-startup-20261102`. No production library was changed. +The test damages only the new copy's database header; it does not reproduce the +field server-session-loss condition. + +The real Mentor opening run produced `MENTOR_DATABASE_LOAD_FAILED`, then: + +- `startup_close_requested` for the launched PCB PID 6532; +- `startup_close_result: CLOSED`; +- worker and supervisor both recorded `process_exit_confirmed`, with no + remaining PCB processes. + +The subsequent separate native Repair run was able to start and completed as +`RESTORED_REVIEW_REQUIRED`, using the fixture's historical backup. This proves +the failed-open session no longer blocks the next repair operation in this +fixture. It does not prove automatic recovery of the field database or establish +that an older backup contains the desired design revision. + +The separate `open-after` run on the same restored fixture completed native +integration, Save/IsSaved/Close/Quit, confirmed process release and returned +`PASS`, `safe_to_continue=True`, `READY (IsVerifiedReuseBlock, raw=1)`. +The `compare` phase passed all seven checks, including absence of the original +opening error and unchanged source-fixture database SHA-256. Its +`server_session_loss_reproduced` field is explicitly `false`. + +Reproduction commands (set `PYTHONPATH=src`; setup requires a new output name): + +```powershell +python tools/validate_icdb_open_repair_reopen.py setup --output issue2-startup-20261102 +python tools/validate_icdb_open_repair_reopen.py open-before --output issue2-startup-20261102 +python tools/validate_icdb_open_repair_reopen.py repair --output issue2-startup-20261102 +python tools/validate_icdb_open_repair_reopen.py open-after --output issue2-startup-20261102 +python tools/validate_icdb_open_repair_reopen.py compare --output issue2-startup-20261102 +``` + +The stages are deliberately separate: the test fixture has a format-corruption +error and historical-backup recovery, not the field lost-session auto-repair +trigger. The test does not change production handling of old-backup recovery. + +READY filtering, hash-scope changes and CBB CRUD are outside this focused fix. diff --git a/src/cbb_refresh_verify/failed_session.py b/src/cbb_refresh_verify/failed_session.py index 76ba18e..b654b49 100644 --- a/src/cbb_refresh_verify/failed_session.py +++ b/src/cbb_refresh_verify/failed_session.py @@ -5,6 +5,116 @@ from pathlib import Path import time +def _command_args(command): + import ctypes + from ctypes import wintypes + shell = ctypes.WinDLL('shell32', use_last_error=True) + kernel = ctypes.WinDLL('kernel32', use_last_error=True) + shell.CommandLineToArgvW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(ctypes.c_int)] + shell.CommandLineToArgvW.restype = ctypes.POINTER(wintypes.LPWSTR) + kernel.LocalFree.argtypes = [wintypes.HLOCAL] + kernel.LocalFree.restype = wintypes.HLOCAL + count = ctypes.c_int() + args = shell.CommandLineToArgvW(command, ctypes.byref(count)) + if not args: + raise ctypes.WinError(ctypes.get_last_error()) + try: + return list(args[:count.value]) + finally: + kernel.LocalFree(ctypes.cast(args, wintypes.HLOCAL)) + + +def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: None, + timeout_seconds=20): + """Normally close our empty PCB after a diagnosed database startup failure. + + No COM attach: there is no document, and the global ROT can name another + session. Unknown windows, live database addresses and other desktops are + retained. WM_CLOSE is cooperative; no save prompt is accepted or killed. + """ + import win32con, win32gui, win32process + from .mentor_adapter import _process_details + from .native_repair_ui import _current_desktop_name + from .icdb_ownership import unreleased_addresses + root = Path(job['root']) + result = {'status': 'RETAINED', 'pcb_pid': pcb_pid} + try: + diagnosed = False + diagnosis_path = root/'diagnostics/icdb-recovery.json' + if diagnosis_path.is_file(): + diagnosis = json.loads(diagnosis_path.read_text('utf-8')) + diagnosed = diagnosis.get('error_code') == 'ICDB_REPAIR_REQUIRED' + dialogs_path = root/'dialogs.jsonl' + if not diagnosed and dialogs_path.is_file(): + for line in dialogs_path.read_text('utf-8').splitlines(): + try: + dialog = json.loads(line) + except ValueError: + continue + if (dialog.get('pid') == pcb_pid and dialog.get('phase') == 'PCB_STARTING' + and dialog.get('error_code') == 'MENTOR_DATABASE_LOAD_FAILED'): + diagnosed = True + if not diagnosed: + raise ValueError('STARTUP_FAILURE_UNCONFIRMED') + desktop = job.get('desktop', '') + if (job.get('interactive_input') or not desktop.startswith('CbbVerify_') + or _current_desktop_name() != desktop): + raise ValueError('STARTUP_DESKTOP_UNCONFIRMED') + if not pcb_pid or pcb_pid in baseline_pids: + raise ValueError('STARTUP_PROCESS_UNCONFIRMED') + expected = os.path.normcase(os.path.abspath(job['pcb'])) + deadline = time.monotonic() + max(0, timeout_seconds) + sent = False + while True: + records = [r for r in _process_details('ExpeditionPCB.exe') if r[0] == pcb_pid] + if not records: + result['status'] = 'CLOSED' + break + _, executable, command, _ = records[0] + args = _command_args(command) + if (Path(executable).name.casefold() != 'expeditionpcb.exe' + or '-reusablecircuit' not in [a.casefold() for a in args] + or expected not in [os.path.normcase(os.path.abspath(a)) for a in args[1:]]): + raise ValueError('STARTUP_PROCESS_UNCONFIRMED') + if not sent: + windows = [] + def observe(hwnd, _): + if (win32process.GetWindowThreadProcessId(hwnd)[1] == pcb_pid + and win32gui.IsWindowVisible(hwnd)): + windows.append(hwnd) + win32gui.EnumWindows(observe, None) + main = [w for w in windows if win32gui.GetClassName(w).startswith('Afx:') + and win32gui.GetWindowText(w).strip() in ( + 'Expedition PCB', 'Expedition PCB - Reusable Block') + and win32gui.IsWindowEnabled(w)] + dialogs = any(win32gui.GetClassName(w) == '#32770' for w in windows) + documents = [] + def child(hwnd, _): + if (win32gui.GetClassName(hwnd) == 'MDIClient' + and win32gui.GetWindow(hwnd, win32con.GW_CHILD)): + documents.append(hwnd) + if len(main) == 1: + win32gui.EnumChildWindows(main[0], child, None) + addresses = unreleased_addresses(job['lmc_file'], job['project'], job['pcb']) + if len(main) == 1 and not dialogs and not documents and not addresses: + win32gui.PostMessage(main[0], win32con.WM_CLOSE, 0, 0) + sent = True + emit(stage='startup_close_requested', pcb_pid=pcb_pid, hwnd=main[0]) + # Observe exit even when the caller requested no extra wait. + continue + result['detail'] = 'STARTUP_IDLE_UNCONFIRMED: 等待空闲窗口和数据库释放' + if time.monotonic() >= deadline: + if sent: + result['detail'] = 'STARTUP_EXIT_UNCONFIRMED: 已请求正常关闭,但 PCB 尚未退出' + break + time.sleep(.2) + except Exception as error: + result['detail'] = str(error) + (root/'startup-close.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') + emit(stage='startup_close_result', **result) + return result + + def close_failed_document(job, *, emit=lambda **event: None, timeout_seconds=30): from .desktop_worker import launch_save_helper root = Path(job['root']) diff --git a/src/cbb_refresh_verify/mentor_adapter.py b/src/cbb_refresh_verify/mentor_adapter.py index d61f524..7925223 100644 --- a/src/cbb_refresh_verify/mentor_adapter.py +++ b/src/cbb_refresh_verify/mentor_adapter.py @@ -863,6 +863,14 @@ def execute_job(job, emit): lm_session.close() except Exception as error: emit(stage='failure_close_unconfirmed', detail=str(error)) + elif not completed and worker_pid is not None: + try: + from .failed_session import close_failed_startup + failed_close = close_failed_startup(job, worker_pid, baseline_pids=pcb_before, emit=emit) + if failed_close.get('status') == 'CLOSED' and lm_session is not None: + lm_session.close() + except Exception as error: + emit(stage='startup_close_unconfirmed', detail=str(error)) cleanup_verify_processes( lm_pid, worker_pid, diff --git a/src/cbb_refresh_verify/verify_runner.py b/src/cbb_refresh_verify/verify_runner.py index 84e95fe..3a0e4a8 100644 --- a/src/cbb_refresh_verify/verify_runner.py +++ b/src/cbb_refresh_verify/verify_runner.py @@ -866,7 +866,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run if dialog_result.error_code: _write_failure_windows(root, latest_snapshots) _write_process_tree(root, process, target_pid) - _stop_worker(process) + # A failed database load has no document to close. Give the + # worker time to normally close its owned empty PCB before + # recovery checks for remaining native database clients. + _stop_worker(process, timeout=40) return finish(VerifyResult( 'FAIL', stderr=f'{dialog_result.error_code}: {dialog_result.detail}', diff --git a/tests/test_failed_startup.py b/tests/test_failed_startup.py new file mode 100644 index 0000000..3fc0b40 --- /dev/null +++ b/tests/test_failed_startup.py @@ -0,0 +1,158 @@ +import json +from pathlib import Path + +import pytest + +from cbb_refresh_verify import failed_session, mentor_adapter + + +def startup_fixture(tmp_path, monkeypatch): + import win32gui, win32process + from cbb_refresh_verify import native_repair_ui + job = {'root': str(tmp_path), 'pcb': str(tmp_path/'PCB/CBB.pcb'), + 'project': str(tmp_path/'CBB.prj'), 'lmc_file': str(tmp_path/'lib.lmc'), + 'desktop': 'CbbVerify_test'} + diagnostics = tmp_path/'diagnostics' + diagnostics.mkdir() + (diagnostics/'icdb-recovery.json').write_text(json.dumps({ + 'error_code': 'ICDB_REPAIR_REQUIRED', 'scope': 'project', + 'database_path': str(tmp_path/'database'), 'session': 'lost-session'})) + records = [(12, r'C:\Mentor\wg\ExpeditionPCB.exe', + f'ExpeditionPCB.exe "{job["pcb"]}" -ReusableCircuit', 11)] + monkeypatch.setattr(mentor_adapter, '_process_details', lambda _: list(records)) + monkeypatch.setattr(native_repair_ui, '_current_desktop_name', lambda: 'CbbVerify_test') + monkeypatch.setattr(win32gui, 'EnumWindows', lambda fn, arg: fn(100, arg)) + monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda *args: None) + monkeypatch.setattr(win32process, 'GetWindowThreadProcessId', lambda _: (1, 12)) + monkeypatch.setattr(win32gui, 'IsWindowVisible', lambda _: True) + monkeypatch.setattr(win32gui, 'IsWindowEnabled', lambda _: True) + monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: 'Expedition PCB') + monkeypatch.setattr(win32gui, 'GetClassName', lambda _: 'Afx:main') + messages = [] + def post(hwnd, message, wparam, lparam): + messages.append((hwnd, message)) + records.clear() + monkeypatch.setattr(win32gui, 'PostMessage', post) + return job, records, messages + + +@pytest.mark.parametrize('title', ['Expedition PCB', 'Expedition PCB - Reusable Block']) +def test_failed_startup_normally_closes_owned_empty_pcb(tmp_path, monkeypatch, title): + import win32con + import win32gui + job, records, messages = startup_fixture(tmp_path, monkeypatch) + monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: title) + result = failed_session.close_failed_startup(job, 12, baseline_pids=(), timeout_seconds=0) + assert result['status'] == 'CLOSED' + assert not records + assert messages == [(100, win32con.WM_CLOSE)] + assert json.loads((tmp_path/'startup-close.json').read_text())['status'] == 'CLOSED' + + +@pytest.mark.parametrize('obstacle', ['baseline', 'wrong_path', 'path_prefix', 'desktop', 'address', + 'dialog', 'document_title', 'mdi_document', 'disabled', 'no_diagnosis']) +def test_failed_startup_preserves_unconfirmed_or_busy_session(tmp_path, monkeypatch, obstacle): + import win32gui + from cbb_refresh_verify import native_repair_ui + job, records, messages = startup_fixture(tmp_path, monkeypatch) + baseline = () + if obstacle == 'baseline': baseline = (12,) + if obstacle == 'wrong_path': records[0] = (12, records[0][1], 'other.pcb -ReusableCircuit', 11) + if obstacle == 'path_prefix': + records[0] = (12, records[0][1], f'pcb.exe "{job["pcb"]}.other" -ReusableCircuit', 11) + if obstacle == 'desktop': monkeypatch.setattr(native_repair_ui, '_current_desktop_name', lambda: 'Default') + if obstacle == 'address': + address = tmp_path/'database/cdbsvr/sAddress.adr' + address.parent.mkdir(parents=True) + address.write_text('owner') + if obstacle == 'dialog': monkeypatch.setattr(win32gui, 'GetClassName', lambda _: '#32770') + if obstacle == 'document_title': monkeypatch.setattr(win32gui, 'GetWindowText', lambda _: 'CBB.pcb - Expedition PCB') + if obstacle == 'mdi_document': + monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda hwnd, fn, arg: fn(101, arg)) + monkeypatch.setattr(win32gui, 'GetClassName', lambda hwnd: 'MDIClient' if hwnd == 101 else 'Afx:main') + monkeypatch.setattr(win32gui, 'GetWindow', lambda *args: 102) + if obstacle == 'disabled': monkeypatch.setattr(win32gui, 'IsWindowEnabled', lambda _: False) + if obstacle == 'no_diagnosis': (tmp_path/'diagnostics/icdb-recovery.json').unlink() + result = failed_session.close_failed_startup(job, 12, baseline_pids=baseline, timeout_seconds=0) + assert result['status'] == 'RETAINED' + assert records + assert not messages + + +def test_requested_close_without_exit_is_not_reported_closed(tmp_path, monkeypatch): + import win32gui + job, records, messages = startup_fixture(tmp_path, monkeypatch) + monkeypatch.setattr(win32gui, 'PostMessage', lambda hwnd, message, *_: messages.append((hwnd, message))) + result = failed_session.close_failed_startup(job, 12, timeout_seconds=0) + assert result['status'] == 'RETAINED' + assert 'STARTUP_EXIT_UNCONFIRMED' in result['detail'] + assert len(messages) == 1 + assert records + + +@pytest.mark.parametrize('obstacle', ['address', 'dialog', 'process']) +def test_startup_close_waits_for_native_state_transitions(tmp_path, monkeypatch, obstacle): + from types import SimpleNamespace + import win32gui + job, records, messages = startup_fixture(tmp_path, monkeypatch) + ticks = [0] + address = tmp_path/'database/cdbsvr/sAddress.adr' + if obstacle == 'address': + address.parent.mkdir(parents=True) + address.write_text('owner') + if obstacle == 'dialog': + monkeypatch.setattr(win32gui, 'GetClassName', + lambda _: '#32770' if ticks[0] == 0 else 'Afx:main') + if obstacle == 'process': + monkeypatch.setattr(win32gui, 'PostMessage', + lambda hwnd, message, *_: messages.append((hwnd, message))) + def advance(seconds): + ticks[0] += seconds + if obstacle == 'address' and address.exists(): address.unlink() + if obstacle == 'process' and messages: records.clear() + monkeypatch.setattr(failed_session, 'time', SimpleNamespace( + monotonic=lambda: ticks[0], sleep=advance)) + result = failed_session.close_failed_startup(job, 12, timeout_seconds=1) + assert ticks[0] > 0 + assert result['status'] == 'CLOSED' + assert len(messages) == 1 + assert not records + + +def test_worker_closes_startup_failure_before_document_identity(tmp_path, monkeypatch): + from test_verify_worker_stability import exercise_worker + job, events, execute = exercise_worker(tmp_path, monkeypatch) + job['desktop'] = 'CbbVerify_test' + def fail_open(*args, **kwargs): + raise RuntimeError('VERIFY_CANCELLED: database open failed') + monkeypatch.setattr('cbb_refresh_verify.pcb_bridge.wait_for_target_document', fail_open) + # Native process/window behavior is covered above; here verify the worker + # actually reaches startup cleanup instead of requiring a loaded document. + def close(job, pid, **kwargs): + kwargs['emit'](stage='startup_close_result', status='CLOSED', pcb_pid=pid) + return {'status': 'CLOSED'} + monkeypatch.setattr(failed_session, 'close_failed_startup', close, raising=False) + with pytest.raises(RuntimeError, match='VERIFY_CANCELLED'): + execute() + assert any(e['stage'] == 'startup_close_result' and e['pcb_pid'] == 2 for e in events) + assert not any(e['stage'] == 'save_start' for e in events) + + +def test_failed_startup_accepts_current_owned_database_failure_dialog(tmp_path, monkeypatch): + job, records, messages = startup_fixture(tmp_path, monkeypatch) + (tmp_path/'diagnostics/icdb-recovery.json').unlink() + (tmp_path/'dialogs.jsonl').write_text(json.dumps({ + 'pid': 12, 'phase': 'PCB_STARTING', 'error_code': 'MENTOR_DATABASE_LOAD_FAILED'})+'\n') + result = failed_session.close_failed_startup(job, 12, timeout_seconds=0) + assert result['status'] == 'CLOSED' + assert not records + + +@pytest.mark.parametrize('pid,phase', [(99, 'PCB_STARTING'), (12, 'SAVING')]) +def test_failed_startup_rejects_unrelated_failure_dialog(tmp_path, monkeypatch, pid, phase): + job, records, messages = startup_fixture(tmp_path, monkeypatch) + (tmp_path/'diagnostics/icdb-recovery.json').unlink() + (tmp_path/'dialogs.jsonl').write_text(json.dumps({ + 'pid': pid, 'phase': phase, 'error_code': 'MENTOR_DATABASE_LOAD_FAILED'})+'\n') + assert failed_session.close_failed_startup(job, 12, timeout_seconds=0)['status'] == 'RETAINED' + assert not messages