fix: supervise management calls across cancellation and parent loss
This commit is contained in:
1 parent
8d3eab1164
commit
6278091f21
8 files changed
+297
-24
No files matched your search
@@ -169,7 +169,7 @@ def _may_have_mutated(record):
|
||||
or any(not isinstance(item, dict) or item.get('stage') not in prewrite for item in history))
|
||||
|
||||
|
||||
def run_management(request, *, adapter=None, verifier=None, idle_check=None, progress=None):
|
||||
def run_management(request, *, adapter=None, verifier=None, idle_check=None, progress=None, stop=None):
|
||||
from .verify_runner import VerifyRequest, run_verify
|
||||
from .backup_recovery import _idle_check
|
||||
import win32con, win32event
|
||||
@@ -191,6 +191,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
change_report = None
|
||||
def invoke(callback, *args):
|
||||
nonlocal owner_safe
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('MANAGEMENT_CANCELLED: 已停止后续原生操作')
|
||||
owner_safe = False
|
||||
try:
|
||||
result = callback(*args)
|
||||
@@ -244,6 +246,10 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
from .com_runtime import initialize_com, uninitialize_com
|
||||
initialized = initialize_com()
|
||||
try:
|
||||
if stop is not None and stop.is_set():
|
||||
return finish('CANCELLED', '已取消,未启动原生管理操作')
|
||||
if hasattr(adapter, 'bind_stop'):
|
||||
adapter.bind_stop(stop)
|
||||
if request.operation not in ('add', 'update', 'remove'):
|
||||
raise ValueError('不支持的管理操作。')
|
||||
validate_name(request.name); validate_name(request.partition)
|
||||
@@ -349,6 +355,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
if build_exclusive_library_manifest(library_root, scope=scope, progress=progress) != original:
|
||||
raise OSError('内容基线记录后中心库发生变化,停止写入。')
|
||||
if request.operation in ('remove', 'update'):
|
||||
if stop is not None and stop.is_set():
|
||||
return finish('CANCELLED', '已取消,未开始移除', owner_safe)
|
||||
checkpoint('native_remove')
|
||||
mutated = True; owner_safe = False
|
||||
native = invoke(adapter.remove, request, root/'remove')
|
||||
@@ -365,6 +373,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
checkpoint('removed')
|
||||
if request.operation == 'remove':
|
||||
return finish('REMOVED', '已由原生操作移除,重新读取目录确认不存在;应用未创建备份。')
|
||||
if stop is not None and stop.is_set():
|
||||
return finish('RECOVERY_REQUIRED' if mutated else 'CANCELLED',
|
||||
'已停止后续导入;已发生的原生变更需要核对', owner_safe)
|
||||
checkpoint('native_add')
|
||||
mutated = True; owner_safe = False
|
||||
native = invoke(adapter.add, staged, root/'add')
|
||||
@@ -391,7 +402,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
checkpoint('verify')
|
||||
owner_safe = False
|
||||
verified = verifier(VerifyRequest(library, request.name, root/'verify',
|
||||
mapped_drive=request.mapped_drive, mapped_remote_root=request.mapped_remote_root), progress=progress)
|
||||
mapped_drive=request.mapped_drive, mapped_remote_root=request.mapped_remote_root), progress=progress, stop=stop)
|
||||
owner_safe = verified.safe_to_continue
|
||||
checkpoint('verified', verification=asdict(verified))
|
||||
good = verified.status == 'PASS' and verified.library_flag == 'GREEN' and owner_safe
|
||||
@@ -417,14 +428,24 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
|
||||
def run_management_batch(requests, *, stop=None, progress=None, **kwargs):
|
||||
results = []
|
||||
retained_libraries = set()
|
||||
for request in requests:
|
||||
if stop is not None and stop.is_set():
|
||||
break
|
||||
result = run_management(request, progress=progress, **kwargs)
|
||||
key = str(request.library).casefold()
|
||||
if key in retained_libraries:
|
||||
result = ManagementResult(request.operation, request.library, request.name, request.partition,
|
||||
request.source, request.auto_verify, 'complete', 'SKIPPED', True,
|
||||
'同库原生会话尚未释放,未启动新的写入')
|
||||
results.append(result)
|
||||
if progress:
|
||||
progress({'stage': result.status, 'block': result.name, 'detail': result.detail})
|
||||
continue
|
||||
result = run_management(request, progress=progress, stop=stop, **kwargs)
|
||||
results.append(result)
|
||||
if progress:
|
||||
progress({'stage': result.status, 'block': result.name, 'detail': result.detail,
|
||||
'evidence_path': str(result.evidence_path or '')})
|
||||
if not result.safe_to_continue:
|
||||
break
|
||||
retained_libraries.add(key)
|
||||
return results
|
||||
@@ -69,9 +69,14 @@ class NativeManagementAdapter:
|
||||
self.mapped_drive = ''
|
||||
self.mapped_remote_root = ''
|
||||
self._pending_workers = {}
|
||||
self.stop = None
|
||||
self.session_progress = None
|
||||
|
||||
def bind_stop(self, stop):
|
||||
self.stop = stop
|
||||
|
||||
def release_confirmed(self, library):
|
||||
"""Reap only this adapter's timed-out worker after its own release result."""
|
||||
"""Worker exit/result files do not replace independent resource proof."""
|
||||
key = str(Path(library).resolve()).casefold()
|
||||
pending = self._pending_workers.get(key)
|
||||
if pending is None:
|
||||
@@ -80,10 +85,17 @@ class NativeManagementAdapter:
|
||||
if worker.poll() is None:
|
||||
return False
|
||||
try:
|
||||
result = json.loads((evidence/'result.json').read_text(encoding='utf-8'))
|
||||
if result.get('safe_to_continue') is not True:
|
||||
return False
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
if (evidence/'session.json').is_file():
|
||||
from .session_record import reconcile_session_record
|
||||
proof = reconcile_session_record(evidence/'session.json', evidence_root=evidence)
|
||||
if proof['status'] != 'RELEASED':
|
||||
return False
|
||||
else:
|
||||
from .library_lock import release_retained_library
|
||||
proof = release_retained_library(library, evidence/'release-proof')
|
||||
if proof.get('status') != 'RELEASE_CONFIRMED':
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
worker.close()
|
||||
if worker in _retained_workers:
|
||||
@@ -208,6 +220,12 @@ class NativeManagementAdapter:
|
||||
|
||||
def _launch_worker_observed(self, job, evidence, state):
|
||||
from .desktop_worker import DesktopWorker
|
||||
from .session_record import process_identity
|
||||
job = {**job, 'cancel_signal': str(evidence/'cancel-management.signal'),
|
||||
'start_gate': str(evidence/'start-management.signal'),
|
||||
'session_record_path': str(evidence/'session.json'), 'operation_id': uuid4().hex,
|
||||
'supervisor_pid': os.getpid(), 'supervisor_identity': process_identity(os.getpid()),
|
||||
'block': job.get('name')}
|
||||
request_path = evidence/'request.json'
|
||||
desktop, timeout = job['desktop'], job['timeout_seconds']
|
||||
deadline = time.monotonic() + timeout
|
||||
@@ -221,13 +239,27 @@ class NativeManagementAdapter:
|
||||
worker = (self.worker_factory or DesktopWorker)(command + ['--management-worker', str(request_path)], desktop)
|
||||
state['worker'] = worker
|
||||
worker.cbb_library = str(Path(job['library']).resolve())
|
||||
if hasattr(worker, 'bind_cancellation'):
|
||||
worker.bind_cancellation(job['cancel_signal'])
|
||||
if hasattr(worker, 'bind_session'):
|
||||
worker.bind_session(job)
|
||||
Path(job['start_gate']).touch()
|
||||
while worker.poll() is None and time.monotonic() < deadline:
|
||||
if self.stop is not None and self.stop.is_set():
|
||||
Path(job['cancel_signal']).touch()
|
||||
break
|
||||
time.sleep(.1)
|
||||
if worker.poll() is None:
|
||||
# DesktopWorker.close() requests a kill without a cancellation file.
|
||||
# Keep this exact worker alive until its native call returns normally.
|
||||
_retained_workers.append(worker)
|
||||
self._pending_workers[str(Path(job['library']).resolve()).casefold()] = (worker, evidence)
|
||||
Path(job['cancel_signal']).touch()
|
||||
if getattr(worker, 'session_record_path', None):
|
||||
from .session_record import handoff_session
|
||||
handoff_session(worker.session_record_path, evidence_root=evidence, process=worker,
|
||||
progress=(lambda value: self.session_progress({'stage': 'session_observation', **value}))
|
||||
if self.session_progress is not None else None)
|
||||
if add_job:
|
||||
(evidence/'cancel-native-add.signal').touch()
|
||||
return {'success': False, 'safe_to_continue': False,
|
||||
@@ -691,7 +723,25 @@ def _probe_add_dialog(app, job, desktop, owned_pids, root, *, library=None):
|
||||
_close_probe_dialogs(desktop, owned_pids)
|
||||
|
||||
|
||||
def _require_management_continuation(job, root):
|
||||
"""Check before a new native call; never abort a call already on the stack."""
|
||||
from .library_lock import _supervisor_alive
|
||||
cancelled = job.get('cancel_signal') and Path(job['cancel_signal']).exists()
|
||||
supervisor = job.get('supervisor_pid')
|
||||
if cancelled or supervisor is None or not _supervisor_alive(supervisor):
|
||||
raise RuntimeError('MANAGEMENT_CANCELLED: 操作已停止或监督器已退出,未提交新的原生调用')
|
||||
if job.get('supervisor_identity'):
|
||||
from .session_record import process_identity, IDENTITY_KEYS
|
||||
actual = process_identity(supervisor)
|
||||
if any(actual[k] != job['supervisor_identity'].get(k) for k in IDENTITY_KEYS):
|
||||
raise RuntimeError('MANAGEMENT_CANCELLED: 监督器身份已变化,未提交新的原生调用')
|
||||
|
||||
|
||||
def _require_add_supervisor(job, root):
|
||||
try:
|
||||
_require_management_continuation(job, root)
|
||||
except RuntimeError as error:
|
||||
raise RuntimeError(f'NATIVE_ADD_CANCELLED: {error}') from error
|
||||
from .library_lock import _supervisor_alive
|
||||
deadline = job.get('native_add_deadline')
|
||||
if ((Path(root)/'cancel-native-add.signal').exists()
|
||||
@@ -1038,10 +1088,15 @@ def _catalog_com(job, root):
|
||||
result = {'success': False, 'safe_to_continue': False}
|
||||
watcher = None
|
||||
try:
|
||||
_require_management_continuation(job, root)
|
||||
from .session_record import update_worker_state
|
||||
update_worker_state(job, 'NATIVE_CALL')
|
||||
app = win32com.client.DispatchEx('LibraryManager.Application')
|
||||
app.Visible = True
|
||||
processes = _process_details('LibraryManager.exe')
|
||||
owned_pids.update(int(p[0]) for p in processes)
|
||||
from .session_record import update_worker_state
|
||||
update_worker_state(job, 'POSTCALL', identities=[('native', pid) for pid in owned_pids])
|
||||
if len(owned_pids) != 1:
|
||||
raise RuntimeError('无法唯一确认本次 Library Manager 进程。')
|
||||
expected_binary = Path(job['mentor_home'])/'lm/win32/bin/LibraryManager.exe'
|
||||
@@ -1051,12 +1106,17 @@ def _catalog_com(job, root):
|
||||
watcher.start()
|
||||
if job['operation'] in ('probe_create_add', 'probe_create'):
|
||||
target = prepare_probe_library(job, root)
|
||||
_require_management_continuation(job, root)
|
||||
update_worker_state(job, 'NATIVE_CALL')
|
||||
library = app.CreateLibrary(str(target), 1)
|
||||
_write_json(root.parent/'probe-owner.json', {'library': str(target), 'token': job['probe_token']})
|
||||
else:
|
||||
if job['operation'] == 'probe_readd':
|
||||
validate_probe_readd(job, root)
|
||||
_require_management_continuation(job, root)
|
||||
update_worker_state(job, 'NATIVE_CALL')
|
||||
library = app.OpenLibrary(str(absolute_preserving_drive(job['library'])))
|
||||
update_worker_state(job, 'POSTCALL')
|
||||
if refused:
|
||||
raise RuntimeError('目录读取需要升级中心库或选择流程;已拒绝,目标未获准自动升级。')
|
||||
if library is None or Path(str(library.FullName)).resolve() != Path(job['library']).resolve():
|
||||
@@ -1067,6 +1127,8 @@ def _catalog_com(job, root):
|
||||
_probe_add_dialog(app, job, desktop, owned_pids, root, library=library)
|
||||
library.Close()
|
||||
library = None
|
||||
_require_management_continuation(job, root)
|
||||
update_worker_state(job, 'NATIVE_CALL')
|
||||
library = app.OpenLibrary(str(absolute_preserving_drive(job['library'])))
|
||||
if (library is None or Path(str(library.FullName)).resolve() != Path(job['library']).resolve()
|
||||
or int(library.FlowType) != 1):
|
||||
@@ -1138,6 +1200,20 @@ def management_worker_main(request_path):
|
||||
_, desktop = _current_desktop()
|
||||
if desktop != job['desktop'] or not desktop.startswith('CbbManagement_'):
|
||||
raise RuntimeError('原生管理操作仅允许在本次私有桌面执行。')
|
||||
_require_management_continuation(job, root)
|
||||
if job.get('cancel_signal'):
|
||||
from .desktop_worker import start_supervisor_watchdog
|
||||
start_supervisor_watchdog(job['supervisor_pid'], cancel_path=job['cancel_signal'],
|
||||
supervisor_identity=job.get('supervisor_identity'),
|
||||
session_record_path=job.get('session_record_path'), operation_id=job.get('operation_id'))
|
||||
if job.get('start_gate'):
|
||||
deadline = time.monotonic() + 30
|
||||
while not Path(job['start_gate']).is_file():
|
||||
_require_management_continuation(job, root)
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('MANAGEMENT_CANCELLED: 启动门未放行,未启动原生操作')
|
||||
time.sleep(.05)
|
||||
_require_management_continuation(job, root)
|
||||
if job.get('mapped_drive') and job.get('mapped_remote_root'):
|
||||
from .network_path import ensure_mapped_drive_visible
|
||||
ensure_mapped_drive_visible(job['mapped_drive'], job['mapped_remote_root'])
|
||||
@@ -1165,6 +1241,9 @@ def management_worker_main(request_path):
|
||||
command = [str(adapter.home/'lm/win32/bin/CleanWGLib.exe'), str(job['library']), 'F4', str(input_path)]
|
||||
_write_json(root/'command.json', command)
|
||||
with (root/'native-output.txt').open('wb') as output:
|
||||
_require_management_continuation(job, root)
|
||||
from .session_record import update_worker_state
|
||||
update_worker_state(job, 'NATIVE_CALL')
|
||||
result['safe_to_continue'] = False
|
||||
try:
|
||||
process = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT,
|
||||
@@ -1172,17 +1251,27 @@ def management_worker_main(request_path):
|
||||
except (FileNotFoundError, PermissionError):
|
||||
result['safe_to_continue'] = True
|
||||
raise
|
||||
_write_json(root/'native-process.json', {'pid': process.pid, 'command': command})
|
||||
update_worker_state(job, 'NATIVE_CALL', identities=[('native', process.pid)])
|
||||
try:
|
||||
_write_json(root/'native-process.json', {'pid': process.pid, 'command': command})
|
||||
except OSError:
|
||||
pass # Keep supervising the native owner even when evidence storage fails.
|
||||
# Never terminate this native database owner on timeout.
|
||||
try:
|
||||
code = process.wait(timeout=float(job['timeout_seconds']))
|
||||
except subprocess.TimeoutExpired:
|
||||
result.update(detail=f'TIMEOUT: CleanWGLib PID {process.pid} 尚未释放')
|
||||
else:
|
||||
result.update(safe_to_continue=True, exit_code=code)
|
||||
text = (root/'native-output.txt').read_bytes().decode('mbcs', errors='replace')
|
||||
result.update(success=removal_output_ok(code, text, job['name'], job['partition']),
|
||||
detail=text)
|
||||
while True:
|
||||
try:
|
||||
code = process.wait(timeout=1)
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
result['wait_exceeded'] = True
|
||||
result.update(safe_to_continue=True, exit_code=code)
|
||||
text = (root/'native-output.txt').read_bytes().decode('mbcs', errors='replace')
|
||||
result.update(success=removal_output_ok(code, text, job['name'], job['partition']),
|
||||
detail=text)
|
||||
else:
|
||||
raise ValueError('未验证的原生管理操作')
|
||||
except Exception as error:
|
||||
@@ -1191,5 +1280,7 @@ def management_worker_main(request_path):
|
||||
result['detail'] = str(error)
|
||||
result['exception'] = _exception_details(error)
|
||||
finally:
|
||||
from .session_record import update_worker_state
|
||||
update_worker_state(job, 'FINISHED' if result.get('safe_to_continue') is True else 'UNCONFIRMED')
|
||||
_write_json(root/'result.json', result)
|
||||
return 0 if result['success'] and result['safe_to_continue'] else 2
|
||||
@@ -241,7 +241,8 @@ def test_timeout_stops_batch_and_persists_ambiguous_native_stage(setup):
|
||||
adapter.remove = timeout
|
||||
results = run_management_batch([request, replace(request, name='Other')], adapter=adapter,
|
||||
idle_check=lambda *_: None)
|
||||
assert len(results) == 1 and not results[0].safe_to_continue
|
||||
assert len(results) == 2 and not results[0].safe_to_continue
|
||||
assert results[1].status == 'SKIPPED'
|
||||
assert results[0].status == 'RECOVERY_REQUIRED'
|
||||
|
||||
|
||||
|
||||
@@ -99,7 +99,8 @@ def test_probe_preloads_uia_before_com_proxy_creation_but_catalog_stays_read_onl
|
||||
monkeypatch.setattr(native, '_probe_add_dialog', editor)
|
||||
monkeypatch.setattr(native, '_wait_native_exit', lambda pids: True)
|
||||
result = native._catalog_native({'operation': operation, 'desktop': 'CbbManagement_test',
|
||||
'library': str(library_path), 'mentor_home': str(tmp_path), 'probe_token': 'fixture'}, root)
|
||||
'library': str(library_path), 'mentor_home': str(tmp_path), 'probe_token': 'fixture',
|
||||
'supervisor_pid': __import__('os').getpid()}, root)
|
||||
assert events[:2] == (['dispatch', 'library-close'] if operation == 'catalog' else ['uia-loaded', 'dispatch'])
|
||||
if operation == 'catalog':
|
||||
assert result['success'] and 'uia-loaded' not in events
|
||||
|
||||
@@ -156,7 +156,8 @@ def test_removal_cli_keeps_mapped_drive_spelling(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify.management import Capability
|
||||
job = {'operation': 'remove', 'library': 'S:\\library\\mentor.lmc', 'name': 'Block',
|
||||
'partition': 'Logical-Physical', 'desktop': 'CbbManagement_fixture',
|
||||
'mentor_home': str(tmp_path), 'timeout_seconds': 10}
|
||||
'mentor_home': str(tmp_path), 'timeout_seconds': 10,
|
||||
'supervisor_pid': __import__('os').getpid()}
|
||||
request = tmp_path/'request.json'; request.write_text(json.dumps(job), encoding='utf-8')
|
||||
monkeypatch.setattr(native, '_current_desktop', lambda: (None, job['desktop']))
|
||||
monkeypatch.setattr(native.NativeManagementAdapter, 'capabilities', lambda self: {'remove': Capability(True, 'fake transport')})
|
||||
|
||||
@@ -179,9 +179,15 @@ def test_batch_cache_binds_exact_library_identity(tmp_path):
|
||||
assert fingerprint_library(first).sha256 != fingerprint_library(second).sha256
|
||||
|
||||
|
||||
def test_timed_out_management_worker_releases_only_after_its_result(tmp_path, monkeypatch):
|
||||
def test_timed_out_management_worker_requires_independent_release_proof(tmp_path, monkeypatch):
|
||||
import cbb_refresh_verify.native_management as native
|
||||
state = {'exit': None}
|
||||
proven = {'released': False}
|
||||
def release(*a, **kw):
|
||||
if not proven['released']:
|
||||
raise OSError('native resources still owned')
|
||||
return {'status': 'RELEASE_CONFIRMED', 'library': str(library)}
|
||||
monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', release)
|
||||
worker = SimpleNamespace(poll=lambda: state['exit'], close=lambda: None)
|
||||
adapter = native.NativeManagementAdapter(mentor_home=tmp_path, worker_factory=lambda *_: worker)
|
||||
library = tmp_path/'lib.lmc'; library.touch()
|
||||
@@ -195,6 +201,8 @@ def test_timed_out_management_worker_releases_only_after_its_result(tmp_path, mo
|
||||
assert adapter.release_confirmed(library) is False
|
||||
(evidence/'result.json').write_text(json.dumps({'success': False, 'safe_to_continue': True}))
|
||||
assert adapter.release_confirmed(tmp_path/'other.lmc') is False
|
||||
assert adapter.release_confirmed(library) is False
|
||||
proven['released'] = True
|
||||
assert adapter.release_confirmed(library) is True
|
||||
assert worker not in native._retained_workers
|
||||
finally:
|
||||
@@ -483,19 +491,21 @@ def test_native_add_passive_postcheck_failure_preserves_release(transaction, mon
|
||||
|
||||
|
||||
@pytest.mark.parametrize('case,safe', [('exit_output_error', True), ('popen_missing', True),
|
||||
('popen_denied', True), ('popen_unknown', False), ('timeout', False)])
|
||||
('popen_denied', True), ('popen_unknown', False), ('timeout', True)])
|
||||
def test_remove_worker_preserves_exit_and_known_creation_rejection(tmp_path, monkeypatch, case, safe):
|
||||
import cbb_refresh_verify.native_management as native
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps({'operation': 'remove', 'desktop': 'CbbManagement_test',
|
||||
'mentor_home': str(tmp_path), 'library': str(tmp_path/'lib.lmc'), 'name': 'Block',
|
||||
'partition': 'Logical-Physical', 'timeout_seconds': 1}), encoding='utf-8')
|
||||
'partition': 'Logical-Physical', 'timeout_seconds': 1,
|
||||
'supervisor_pid': __import__('os').getpid()}), encoding='utf-8')
|
||||
monkeypatch.setattr(native, '_current_desktop', lambda: (None, 'CbbManagement_test'))
|
||||
monkeypatch.setattr(native.NativeManagementAdapter, 'capabilities', lambda _: {
|
||||
'remove': SimpleNamespace(supported=True)})
|
||||
observed = {'waited': False}
|
||||
def wait(timeout):
|
||||
if case == 'timeout':
|
||||
if case == 'timeout' and not observed.get('timed_out'):
|
||||
observed['timed_out'] = True
|
||||
raise native.subprocess.TimeoutExpired('fake CleanWGLib', timeout)
|
||||
observed['waited'] = True
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
from dataclasses import replace
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
|
||||
from cbb_refresh_verify import native_management as native
|
||||
from cbb_refresh_verify.session_record import process_identity
|
||||
from test_management import setup, execute
|
||||
|
||||
|
||||
@pytest.mark.parametrize('operation', ['catalog', 'add', 'remove'])
|
||||
@pytest.mark.parametrize('reason', ['cancelled', 'parent_gone', 'identity_changed'])
|
||||
def test_management_worker_checks_stop_before_any_native_route(tmp_path, monkeypatch, operation, reason):
|
||||
cancel = tmp_path/'cancel.signal'
|
||||
identity = process_identity(os.getpid())
|
||||
if reason == 'cancelled':
|
||||
cancel.touch()
|
||||
if reason == 'identity_changed':
|
||||
identity['created_handle'] = 'old'
|
||||
job = dict(operation=operation, library=str(tmp_path/'lib.lmc'), desktop='CbbManagement_Test',
|
||||
supervisor_pid=-1 if reason == 'parent_gone' else os.getpid(),
|
||||
supervisor_identity=identity, cancel_signal=str(cancel), operation_id='test-management')
|
||||
request = tmp_path/'request.json'
|
||||
request.write_text(json.dumps(job))
|
||||
monkeypatch.setattr(native, '_current_desktop', lambda: (None, job['desktop']))
|
||||
monkeypatch.setattr(native, '_catalog_native', lambda *a: pytest.fail('unsupervised native route'))
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', lambda *a, **kw: pytest.fail('unsupervised remove'))
|
||||
assert native.management_worker_main(request) == 2
|
||||
result = json.loads((tmp_path/'result.json').read_text())
|
||||
assert not result['success']
|
||||
assert result['safe_to_continue'] is True
|
||||
assert 'CANCELLED' in result['detail'] or '监督器' in result['detail']
|
||||
|
||||
|
||||
def test_cancel_after_remove_stops_update_before_catalog_and_add(setup):
|
||||
req, adapter, source = setup
|
||||
stop = Event()
|
||||
remove = adapter.remove
|
||||
def stop_after_remove(*args):
|
||||
result = remove(*args)
|
||||
stop.set()
|
||||
return result
|
||||
adapter.remove = stop_after_remove
|
||||
result = execute(replace(req, operation='update', source=source), adapter, stop=stop)
|
||||
assert adapter.calls == ['remove']
|
||||
assert result.status == 'RECOVERY_REQUIRED'
|
||||
assert result.safe_to_continue is True
|
||||
checkpoint = json.loads(result.evidence_path.read_text('utf-8'))
|
||||
assert any(item['stage'] == 'native_remove' for item in checkpoint['history'])
|
||||
|
||||
|
||||
def test_cancel_before_catalog_does_not_launch_native(setup):
|
||||
req, adapter, _ = setup
|
||||
stop = Event()
|
||||
stop.set()
|
||||
adapter.catalog = lambda *a: pytest.fail('catalog after cancellation')
|
||||
result = execute(req, adapter, stop=stop)
|
||||
assert result.status == 'CANCELLED'
|
||||
assert result.safe_to_continue is True
|
||||
|
||||
|
||||
def test_real_management_worker_parent_loss_waits_for_native_boundary(tmp_path):
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from cbb_refresh_verify.session_record import write_session_record
|
||||
supervisor = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(30)'])
|
||||
request, script = tmp_path/'request.json', tmp_path/'worker.py'
|
||||
gate, cancel, started, continued = [tmp_path/name for name in ('gate', 'cancel', 'started', 'continued')]
|
||||
job = dict(operation='catalog', library=str(tmp_path/'lib.lmc'), desktop='CbbManagement_test',
|
||||
supervisor_pid=supervisor.pid, supervisor_identity=process_identity(supervisor.pid),
|
||||
cancel_signal=str(cancel), start_gate=str(gate), operation_id='real-management',
|
||||
session_record_path=str(tmp_path/'session.json'))
|
||||
request.write_text(json.dumps(job))
|
||||
script.write_text('import time,sys\nfrom pathlib import Path\n'
|
||||
'from cbb_refresh_verify import native_management as n\n'
|
||||
'n._current_desktop=lambda:(None,"CbbManagement_test")\n'
|
||||
'def native_call(job,root):\n'
|
||||
f' Path({str(started)!r}).touch()\n'
|
||||
' time.sleep(1.5)\n'
|
||||
' try: n._require_management_continuation(job,root)\n'
|
||||
' except RuntimeError as e: return dict(success=False,safe_to_continue=True,detail=str(e))\n'
|
||||
f' Path({str(continued)!r}).touch()\n'
|
||||
' return dict(success=True,safe_to_continue=True)\n'
|
||||
'n._catalog_native=native_call\n'
|
||||
f'sys.exit(n.management_worker_main({str(request)!r}))\n')
|
||||
worker = subprocess.Popen([sys.executable, str(script)])
|
||||
try:
|
||||
write_session_record(tmp_path/'session.json', dict(operation_id='real-management',
|
||||
library=job['library'], processes=[{'role': 'worker', **process_identity(worker.pid)}]))
|
||||
gate.touch()
|
||||
deadline = time.monotonic()+8
|
||||
while not started.exists() and time.monotonic()<deadline:
|
||||
time.sleep(.02)
|
||||
assert started.exists()
|
||||
supervisor.terminate(); supervisor.wait(5)
|
||||
deadline = time.monotonic()+1
|
||||
while not cancel.exists() and time.monotonic()<deadline:
|
||||
time.sleep(.02)
|
||||
assert cancel.exists() and worker.poll() is None
|
||||
assert worker.wait(5) == 2
|
||||
assert not continued.exists()
|
||||
assert (tmp_path/'supervisor-lost.json').is_file()
|
||||
assert json.loads((tmp_path/'result.json').read_text())['safe_to_continue'] is True
|
||||
finally:
|
||||
for process in (worker, supervisor):
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
process.wait(5)
|
||||
|
||||
|
||||
def test_remove_timeout_keeps_worker_observing_until_native_returns(tmp_path, monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
from threading import Thread
|
||||
entered, release, done = Event(), Event(), Event()
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps(dict(operation='remove', library=str(tmp_path/'lib.lmc'),
|
||||
desktop='CbbManagement_test', mentor_home=str(tmp_path), name='A',
|
||||
partition='Logical-Physical', timeout_seconds=0, supervisor_pid=os.getpid())))
|
||||
monkeypatch.setattr(native, '_current_desktop', lambda: (None, 'CbbManagement_test'))
|
||||
monkeypatch.setattr(native.NativeManagementAdapter, 'capabilities',
|
||||
lambda _: {'remove': SimpleNamespace(supported=True)})
|
||||
def wait(timeout):
|
||||
entered.set()
|
||||
if not release.wait(.05):
|
||||
raise native.subprocess.TimeoutExpired('fake native remove', timeout)
|
||||
return 0
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', lambda *a, **kw: SimpleNamespace(pid=123456, wait=wait))
|
||||
def run():
|
||||
try:
|
||||
native.management_worker_main(job)
|
||||
finally:
|
||||
done.set()
|
||||
thread = Thread(target=run)
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(3)
|
||||
assert not done.wait(.2), 'native timeout abandoned the still-running owner'
|
||||
finally:
|
||||
release.set()
|
||||
thread.join(3)
|
||||
assert done.is_set()
|
||||
assert json.loads((tmp_path/'result.json').read_text())['safe_to_continue'] is True
|
||||
@@ -384,7 +384,8 @@ def test_native_catalog_only_uses_com_records_and_checks_exact_process(tmp_path,
|
||||
monkeypatch.setattr(mentor, '_process_details', lambda _: [(123, str(tmp_path/'lm/win32/bin/LibraryManager.exe'), '', 0)] if state['running'] else [])
|
||||
monkeypatch.setattr(native, '_wait_native_exit', lambda _: not state['running'])
|
||||
result = native._catalog_native({'operation': 'catalog', 'library': str(library_path),
|
||||
'desktop': 'CbbManagement_test', 'mentor_home': str(tmp_path)}, tmp_path)
|
||||
'desktop': 'CbbManagement_test', 'mentor_home': str(tmp_path),
|
||||
'supervisor_pid': __import__('os').getpid()}, tmp_path)
|
||||
assert result['success'] and result['safe_to_continue']
|
||||
assert [(entry['name'], entry['partition']) for entry in result['entries']] == records
|
||||
assert result['flow'] == flow
|
||||
@@ -461,7 +462,8 @@ def test_shared_add_reopens_exact_catalog_and_reports_identity_mismatch(tmp_path
|
||||
monkeypatch.setattr(native, 'validate_owned_fixture_add', lambda *a: None)
|
||||
result = native._catalog_com({'operation': 'add', 'library': str(library_path),
|
||||
'desktop': 'CbbManagement_test', 'mentor_home': str(tmp_path),
|
||||
'name': 'Block', 'partition': 'Logical-Physical', 'source_identity': {'expected': 'bytes'}}, tmp_path)
|
||||
'name': 'Block', 'partition': 'Logical-Physical', 'source_identity': {'expected': 'bytes'},
|
||||
'supervisor_pid': __import__('os').getpid()}, tmp_path)
|
||||
assert events == ['open', 'add', 'close', 'open', 'catalog', 'close']
|
||||
assert result['success'] is registered
|
||||
assert result['identity_matches'] is False
|
||||
|
||||
Reference in new issue
Block a user