fix: report hidden worker admission failures without frozen error dialogs
This commit is contained in:
1 parent
f964ff347c
commit
8bfa4eed34
5 files changed
+160
-21
No files matched your search
@@ -72,6 +72,65 @@ def _positive_int(value):
|
||||
return number
|
||||
|
||||
|
||||
def _native_worker_entry(mode, request):
|
||||
"""Contain hidden-worker failures so a frozen bootloader cannot show a modal error."""
|
||||
import contextlib
|
||||
import json
|
||||
import traceback
|
||||
from .desktop_worker import run_native_worker
|
||||
state = {'worker_entered': False}
|
||||
result_name = {'verify': 'outcome.json', 'repair': 'repair-result.json'}.get(mode, 'result.json')
|
||||
try:
|
||||
with request.with_name('worker.log').open('a', encoding='utf-8') as log:
|
||||
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log):
|
||||
if mode == 'native-backup':
|
||||
from .native_backup import run_native_backup_worker
|
||||
return run_native_backup_worker(request, admission_state=state)
|
||||
if mode == 'management':
|
||||
from .native_management import catalog_worker_main, management_worker_main
|
||||
job = json.loads(request.read_text(encoding='utf-8'))
|
||||
action = catalog_worker_main if job.get('worker_role') == 'catalog_readonly' else management_worker_main
|
||||
elif mode == 'repair':
|
||||
from .native_repair_ui import repair_worker_main
|
||||
action = repair_worker_main
|
||||
else:
|
||||
from .mentor_adapter import worker_main
|
||||
action = worker_main
|
||||
return run_native_worker(action, request, admission_state=state)
|
||||
except Exception as error:
|
||||
trace = traceback.format_exc()
|
||||
entered = state['worker_entered']
|
||||
reason = str(error)
|
||||
cancelled = not entered and reason.startswith('WORKER_CANCELLED:')
|
||||
detail = ('VERIFY_CANCELLED: ' if mode == 'verify' and cancelled else '') + reason
|
||||
path = request.with_name(result_name)
|
||||
try:
|
||||
result = json.loads(path.read_text(encoding='utf-8'))
|
||||
if not isinstance(result, dict): result = {}
|
||||
except (OSError, ValueError):
|
||||
result = {}
|
||||
if mode == 'verify':
|
||||
result.update(status='CANCELLED' if cancelled else 'FAIL', stderr=detail,
|
||||
error_stage='worker' if entered else 'admission', safe_to_continue=False)
|
||||
else:
|
||||
result.update(success=False, status='FAILED', detail=detail, safe_to_continue=False)
|
||||
failure = dict(stage='worker' if entered else 'admission', reason=reason,
|
||||
worker_entered=entered, native_started=None if entered or mode == 'native-backup' else False,
|
||||
safe_to_continue=False)
|
||||
# An exception is never a release proof, and existing Save/Close evidence is retained.
|
||||
for output, value in ((path, result), (request.with_name('worker-failure.json'), failure)):
|
||||
try:
|
||||
output.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
with request.with_name('worker.log').open('a', encoding='utf-8') as log:
|
||||
log.write(trace)
|
||||
except OSError:
|
||||
pass
|
||||
return 2 if mode == 'management' else 1
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="CbbRefreshVerify")
|
||||
parser.add_argument('--verify-worker', type=Path, help=argparse.SUPPRESS)
|
||||
@@ -97,19 +156,11 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
from .owner_probe import owner_probe_main
|
||||
return owner_probe_main(args.owner_probe)
|
||||
if args.native_backup_worker:
|
||||
from .native_backup import run_native_backup_worker
|
||||
return run_native_backup_worker(args.native_backup_worker)
|
||||
return _native_worker_entry('native-backup', args.native_backup_worker)
|
||||
if args.management_worker:
|
||||
import json
|
||||
from .native_management import catalog_worker_main, management_worker_main
|
||||
from .desktop_worker import run_native_worker
|
||||
job = json.loads(args.management_worker.read_text(encoding='utf-8'))
|
||||
action = catalog_worker_main if job.get('worker_role') == 'catalog_readonly' else management_worker_main
|
||||
return run_native_worker(action, args.management_worker)
|
||||
return _native_worker_entry('management', args.management_worker)
|
||||
if args.repair_worker:
|
||||
from .native_repair_ui import repair_worker_main
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(repair_worker_main, args.repair_worker)
|
||||
return _native_worker_entry('repair', args.repair_worker)
|
||||
if args.collect_failure_logs:
|
||||
from .verify_diagnostics import failure_logs_main
|
||||
return failure_logs_main(args.collect_failure_logs)
|
||||
@@ -117,12 +168,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
from .dialog_agent import dialog_agent_main
|
||||
return dialog_agent_main(args.dialog_agent)
|
||||
if args.verify_worker:
|
||||
from .mentor_adapter import worker_main
|
||||
import contextlib
|
||||
with args.verify_worker.with_name('worker.log').open('a', encoding='utf-8') as log:
|
||||
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log):
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(worker_main, args.verify_worker)
|
||||
return _native_worker_entry('verify', args.verify_worker)
|
||||
context = configure(resolve_data_root(args.data_root)); logger = context.logger; logger.info("application_start")
|
||||
try:
|
||||
if args.batch_lmc:
|
||||
|
||||
@@ -183,7 +183,7 @@ def worker_native_channel(*, lease_handle=None, inspection_only=False):
|
||||
win32api.CloseHandle(int(lease_handle))
|
||||
|
||||
|
||||
def run_native_worker(action, request_path):
|
||||
def run_native_worker(action, request_path, *, admission_state=None):
|
||||
"""Wait for durable parent admission before acquiring the worker barrier."""
|
||||
import json
|
||||
import time
|
||||
@@ -232,6 +232,8 @@ def run_native_worker(action, request_path):
|
||||
with worker_native_channel(lease_handle=lease_handle, inspection_only=inspection):
|
||||
if job.get('cancel_signal') and Path(job['cancel_signal']).exists():
|
||||
raise RuntimeError('WORKER_CANCELLED: 已取消原生启动。')
|
||||
if admission_state is not None:
|
||||
admission_state['worker_entered'] = True
|
||||
return action(request_path)
|
||||
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ class _NativeBackupProcess:
|
||||
self.handle.Close(); self.job.Close()
|
||||
|
||||
|
||||
def run_native_backup_worker(job):
|
||||
def run_native_backup_worker(job, *, admission_state=None):
|
||||
"""Publish admission refusal only when this exact worker never entered native work."""
|
||||
from .desktop_worker import run_native_worker
|
||||
from .management import _write_json
|
||||
@@ -212,7 +212,7 @@ def run_native_backup_worker(job):
|
||||
entered = True
|
||||
return native_backup_worker_main(path)
|
||||
try:
|
||||
return run_native_worker(execute, job)
|
||||
return run_native_worker(execute, job, admission_state=admission_state)
|
||||
except Exception as error:
|
||||
bound = registered()
|
||||
_write_json(job.with_name('result.json'), {'status': 'FAILED', 'detail': str(error),
|
||||
|
||||
@@ -273,7 +273,7 @@ def test_unbound_admission_refusal_cannot_claim_release(tmp_path, monkeypatch):
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps(dict(library=str(tmp_path/'lib.lmc'), block='A', operation_id='foreign',
|
||||
session_record_path=str(tmp_path/'session.json'), project=str(tmp_path/'A.prj'))))
|
||||
monkeypatch.setattr(desktop_worker, 'run_native_worker', lambda *a: (_ for _ in ()).throw(RuntimeError('refused')))
|
||||
monkeypatch.setattr(desktop_worker, 'run_native_worker', lambda *a, admission_state=None: (_ for _ in ()).throw(RuntimeError('refused')))
|
||||
assert native.run_native_backup_worker(job) == 1
|
||||
result = json.loads(job.with_name('result.json').read_text('utf-8'))
|
||||
assert result['safe_to_continue'] is False
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Hidden native workers must publish errors instead of a frozen bootloader popup."""
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh_worker_registry(monkeypatch):
|
||||
from cbb_refresh_verify import desktop_worker
|
||||
# Each hidden entry runs in a new process, without earlier tests' fake clients.
|
||||
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mode,result_file', [
|
||||
('verify', 'outcome.json'), ('management', 'result.json'),
|
||||
('management_catalog', 'result.json'), ('repair', 'repair-result.json')])
|
||||
@pytest.mark.parametrize('reason', ['cancelled', 'busy', 'invalid_session', 'mismatched_session', 'invalid_json'])
|
||||
def test_hidden_worker_admission_refusal_exits_with_unconfirmed_receipt(tmp_path, monkeypatch, mode, result_file, reason):
|
||||
from cbb_refresh_verify import application, desktop_worker
|
||||
request = tmp_path/'request.json'
|
||||
job = {'root': str(tmp_path), 'block_name': 'Fixture', 'operation': 'catalog'}
|
||||
if mode == 'management_catalog':
|
||||
mode = 'management'
|
||||
job['worker_role'] = 'catalog_readonly'
|
||||
if reason == 'cancelled':
|
||||
cancel = tmp_path/'cancel.signal'; cancel.touch()
|
||||
job['cancel_signal'] = str(cancel)
|
||||
elif reason == 'invalid_session':
|
||||
job['session_record_path'] = str(tmp_path/'missing-session.json')
|
||||
elif reason == 'mismatched_session':
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from cbb_refresh_verify.session_record import register_worker_session
|
||||
job.update(session_record_path=str(tmp_path/'session.json'), operation_id='admission',
|
||||
library=str(tmp_path/'library.lmc'), block='Fixture')
|
||||
register_worker_session(SimpleNamespace(pid=os.getpid()), job)
|
||||
job['library'] = str(tmp_path/'different-library.lmc')
|
||||
elif reason == 'busy':
|
||||
@contextmanager
|
||||
def occupied(**kwargs):
|
||||
raise RuntimeError('NATIVE_WRITER_ACTIVE: independent writer')
|
||||
yield
|
||||
monkeypatch.setattr(desktop_worker, 'worker_native_channel', occupied)
|
||||
request.write_text('{' if reason == 'invalid_json' else json.dumps(job), encoding='utf-8')
|
||||
assert application.main(['--'+mode+'-worker', str(request)]) != 0
|
||||
result = json.loads(request.with_name(result_file).read_text('utf-8'))
|
||||
assert result['safe_to_continue'] is False
|
||||
failure = json.loads(request.with_name('worker-failure.json').read_text('utf-8'))
|
||||
assert failure['stage'] == 'admission' and failure['worker_entered'] is False
|
||||
assert failure['native_started'] is False
|
||||
assert 'Traceback' in request.with_name('worker.log').read_text('utf-8')
|
||||
assert not (tmp_path/'save.log').exists()
|
||||
|
||||
|
||||
def test_entered_worker_failure_preserves_existing_save_facts_and_stays_unconfirmed(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify import application, mentor_adapter
|
||||
request = tmp_path/'request.json'; request.write_text(json.dumps({'root': str(tmp_path)}))
|
||||
facts = {'save_state': 'COMMITTED', 'close_state': 'UNKNOWN', 'helper_state': 'RUNNING'}
|
||||
state = tmp_path/'worker-state.json'
|
||||
def entered(path):
|
||||
state.write_text(json.dumps({'write_outcome': facts}))
|
||||
(tmp_path/'outcome.json').write_text(json.dumps({'status': 'FAIL', 'safe_to_continue': False,
|
||||
'write_outcome': facts, 'stderr': 'save returned; close still pending'}))
|
||||
raise RuntimeError('post-entry transport failed')
|
||||
monkeypatch.setattr(mentor_adapter, 'worker_main', entered)
|
||||
assert application.main(['--verify-worker', str(request)]) != 0
|
||||
result = json.loads((tmp_path/'outcome.json').read_text())
|
||||
assert result['write_outcome'] == facts and result['safe_to_continue'] is False
|
||||
assert json.loads(state.read_text()) == {'write_outcome': facts}
|
||||
failure = json.loads((tmp_path/'worker-failure.json').read_text())
|
||||
assert failure['stage'] == 'worker' and failure['worker_entered'] is True
|
||||
assert failure['native_started'] is None
|
||||
|
||||
|
||||
def test_native_backup_bootstrap_error_also_exits_without_claiming_release(tmp_path):
|
||||
from cbb_refresh_verify.application import main
|
||||
request = tmp_path/'request.json'; request.write_text('{')
|
||||
assert main(['--native-backup-worker', str(request)]) != 0
|
||||
assert json.loads((tmp_path/'result.json').read_text())['safe_to_continue'] is False
|
||||
|
||||
|
||||
def test_receipt_io_failure_still_cannot_escape_to_frozen_bootloader(tmp_path, monkeypatch):
|
||||
from pathlib import Path
|
||||
from cbb_refresh_verify.application import main
|
||||
request = tmp_path/'request.json'; request.write_text('{')
|
||||
def denied(*args, **kwargs):
|
||||
raise PermissionError('receipt disk unavailable')
|
||||
monkeypatch.setattr(Path, 'write_text', denied)
|
||||
assert main(['--verify-worker', str(request)]) == 1
|
||||
assert 'JSONDecodeError' in (tmp_path/'worker.log').read_text()
|
||||
Reference in new issue
Block a user