diff --git a/src/cbb_refresh_verify/management.py b/src/cbb_refresh_verify/management.py index bcd7266..cde7ea9 100644 --- a/src/cbb_refresh_verify/management.py +++ b/src/cbb_refresh_verify/management.py @@ -169,7 +169,7 @@ def _may_have_mutated(record): or any(not isinstance(item, dict) or item.get('stage') not in prewrite for item in history)) -def run_management(request, *, adapter=None, verifier=None, idle_check=None, progress=None): +def run_management(request, *, adapter=None, verifier=None, idle_check=None, progress=None, stop=None): from .verify_runner import VerifyRequest, run_verify from .backup_recovery import _idle_check import win32con, win32event @@ -191,6 +191,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro change_report = None def invoke(callback, *args): nonlocal owner_safe + if stop is not None and stop.is_set(): + raise InterruptedError('MANAGEMENT_CANCELLED: 已停止后续原生操作') owner_safe = False try: result = callback(*args) @@ -244,6 +246,10 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro from .com_runtime import initialize_com, uninitialize_com initialized = initialize_com() try: + if stop is not None and stop.is_set(): + return finish('CANCELLED', '已取消,未启动原生管理操作') + if hasattr(adapter, 'bind_stop'): + adapter.bind_stop(stop) if request.operation not in ('add', 'update', 'remove'): raise ValueError('不支持的管理操作。') validate_name(request.name); validate_name(request.partition) @@ -349,6 +355,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro if build_exclusive_library_manifest(library_root, scope=scope, progress=progress) != original: raise OSError('内容基线记录后中心库发生变化,停止写入。') if request.operation in ('remove', 'update'): + if stop is not None and stop.is_set(): + return finish('CANCELLED', '已取消,未开始移除', owner_safe) checkpoint('native_remove') mutated = True; owner_safe = False native = invoke(adapter.remove, request, root/'remove') @@ -365,6 +373,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro checkpoint('removed') if request.operation == 'remove': return finish('REMOVED', '已由原生操作移除,重新读取目录确认不存在;应用未创建备份。') + if stop is not None and stop.is_set(): + return finish('RECOVERY_REQUIRED' if mutated else 'CANCELLED', + '已停止后续导入;已发生的原生变更需要核对', owner_safe) checkpoint('native_add') mutated = True; owner_safe = False native = invoke(adapter.add, staged, root/'add') @@ -391,7 +402,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro checkpoint('verify') owner_safe = False verified = verifier(VerifyRequest(library, request.name, root/'verify', - mapped_drive=request.mapped_drive, mapped_remote_root=request.mapped_remote_root), progress=progress) + mapped_drive=request.mapped_drive, mapped_remote_root=request.mapped_remote_root), progress=progress, stop=stop) owner_safe = verified.safe_to_continue checkpoint('verified', verification=asdict(verified)) good = verified.status == 'PASS' and verified.library_flag == 'GREEN' and owner_safe @@ -417,14 +428,24 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro def run_management_batch(requests, *, stop=None, progress=None, **kwargs): results = [] + retained_libraries = set() for request in requests: if stop is not None and stop.is_set(): break - result = run_management(request, progress=progress, **kwargs) + key = str(request.library).casefold() + if key in retained_libraries: + result = ManagementResult(request.operation, request.library, request.name, request.partition, + request.source, request.auto_verify, 'complete', 'SKIPPED', True, + '同库原生会话尚未释放,未启动新的写入') + results.append(result) + if progress: + progress({'stage': result.status, 'block': result.name, 'detail': result.detail}) + continue + result = run_management(request, progress=progress, stop=stop, **kwargs) results.append(result) if progress: progress({'stage': result.status, 'block': result.name, 'detail': result.detail, 'evidence_path': str(result.evidence_path or '')}) if not result.safe_to_continue: - break + retained_libraries.add(key) return results diff --git a/src/cbb_refresh_verify/native_management.py b/src/cbb_refresh_verify/native_management.py index fd3a360..ed7fa98 100644 --- a/src/cbb_refresh_verify/native_management.py +++ b/src/cbb_refresh_verify/native_management.py @@ -69,9 +69,14 @@ class NativeManagementAdapter: self.mapped_drive = '' self.mapped_remote_root = '' self._pending_workers = {} + self.stop = None + self.session_progress = None + + def bind_stop(self, stop): + self.stop = stop def release_confirmed(self, library): - """Reap only this adapter's timed-out worker after its own release result.""" + """Worker exit/result files do not replace independent resource proof.""" key = str(Path(library).resolve()).casefold() pending = self._pending_workers.get(key) if pending is None: @@ -80,10 +85,17 @@ class NativeManagementAdapter: if worker.poll() is None: return False try: - result = json.loads((evidence/'result.json').read_text(encoding='utf-8')) - if result.get('safe_to_continue') is not True: - return False - except (OSError, ValueError, TypeError, AttributeError): + if (evidence/'session.json').is_file(): + from .session_record import reconcile_session_record + proof = reconcile_session_record(evidence/'session.json', evidence_root=evidence) + if proof['status'] != 'RELEASED': + return False + else: + from .library_lock import release_retained_library + proof = release_retained_library(library, evidence/'release-proof') + if proof.get('status') != 'RELEASE_CONFIRMED': + return False + except Exception: return False worker.close() if worker in _retained_workers: @@ -208,6 +220,12 @@ class NativeManagementAdapter: def _launch_worker_observed(self, job, evidence, state): from .desktop_worker import DesktopWorker + from .session_record import process_identity + job = {**job, 'cancel_signal': str(evidence/'cancel-management.signal'), + 'start_gate': str(evidence/'start-management.signal'), + 'session_record_path': str(evidence/'session.json'), 'operation_id': uuid4().hex, + 'supervisor_pid': os.getpid(), 'supervisor_identity': process_identity(os.getpid()), + 'block': job.get('name')} request_path = evidence/'request.json' desktop, timeout = job['desktop'], job['timeout_seconds'] deadline = time.monotonic() + timeout @@ -221,13 +239,27 @@ class NativeManagementAdapter: worker = (self.worker_factory or DesktopWorker)(command + ['--management-worker', str(request_path)], desktop) state['worker'] = worker worker.cbb_library = str(Path(job['library']).resolve()) + if hasattr(worker, 'bind_cancellation'): + worker.bind_cancellation(job['cancel_signal']) + if hasattr(worker, 'bind_session'): + worker.bind_session(job) + Path(job['start_gate']).touch() while worker.poll() is None and time.monotonic() < deadline: + if self.stop is not None and self.stop.is_set(): + Path(job['cancel_signal']).touch() + break time.sleep(.1) if worker.poll() is None: # DesktopWorker.close() requests a kill without a cancellation file. # Keep this exact worker alive until its native call returns normally. _retained_workers.append(worker) self._pending_workers[str(Path(job['library']).resolve()).casefold()] = (worker, evidence) + Path(job['cancel_signal']).touch() + if getattr(worker, 'session_record_path', None): + from .session_record import handoff_session + handoff_session(worker.session_record_path, evidence_root=evidence, process=worker, + progress=(lambda value: self.session_progress({'stage': 'session_observation', **value})) + if self.session_progress is not None else None) if add_job: (evidence/'cancel-native-add.signal').touch() return {'success': False, 'safe_to_continue': False, @@ -691,7 +723,25 @@ def _probe_add_dialog(app, job, desktop, owned_pids, root, *, library=None): _close_probe_dialogs(desktop, owned_pids) +def _require_management_continuation(job, root): + """Check before a new native call; never abort a call already on the stack.""" + from .library_lock import _supervisor_alive + cancelled = job.get('cancel_signal') and Path(job['cancel_signal']).exists() + supervisor = job.get('supervisor_pid') + if cancelled or supervisor is None or not _supervisor_alive(supervisor): + raise RuntimeError('MANAGEMENT_CANCELLED: 操作已停止或监督器已退出,未提交新的原生调用') + if job.get('supervisor_identity'): + from .session_record import process_identity, IDENTITY_KEYS + actual = process_identity(supervisor) + if any(actual[k] != job['supervisor_identity'].get(k) for k in IDENTITY_KEYS): + raise RuntimeError('MANAGEMENT_CANCELLED: 监督器身份已变化,未提交新的原生调用') + + def _require_add_supervisor(job, root): + try: + _require_management_continuation(job, root) + except RuntimeError as error: + raise RuntimeError(f'NATIVE_ADD_CANCELLED: {error}') from error from .library_lock import _supervisor_alive deadline = job.get('native_add_deadline') if ((Path(root)/'cancel-native-add.signal').exists() @@ -1038,10 +1088,15 @@ def _catalog_com(job, root): result = {'success': False, 'safe_to_continue': False} watcher = None try: + _require_management_continuation(job, root) + from .session_record import update_worker_state + update_worker_state(job, 'NATIVE_CALL') app = win32com.client.DispatchEx('LibraryManager.Application') app.Visible = True processes = _process_details('LibraryManager.exe') owned_pids.update(int(p[0]) for p in processes) + from .session_record import update_worker_state + update_worker_state(job, 'POSTCALL', identities=[('native', pid) for pid in owned_pids]) if len(owned_pids) != 1: raise RuntimeError('无法唯一确认本次 Library Manager 进程。') expected_binary = Path(job['mentor_home'])/'lm/win32/bin/LibraryManager.exe' @@ -1051,12 +1106,17 @@ def _catalog_com(job, root): watcher.start() if job['operation'] in ('probe_create_add', 'probe_create'): target = prepare_probe_library(job, root) + _require_management_continuation(job, root) + update_worker_state(job, 'NATIVE_CALL') library = app.CreateLibrary(str(target), 1) _write_json(root.parent/'probe-owner.json', {'library': str(target), 'token': job['probe_token']}) else: if job['operation'] == 'probe_readd': validate_probe_readd(job, root) + _require_management_continuation(job, root) + update_worker_state(job, 'NATIVE_CALL') library = app.OpenLibrary(str(absolute_preserving_drive(job['library']))) + update_worker_state(job, 'POSTCALL') if refused: raise RuntimeError('目录读取需要升级中心库或选择流程;已拒绝,目标未获准自动升级。') if library is None or Path(str(library.FullName)).resolve() != Path(job['library']).resolve(): @@ -1067,6 +1127,8 @@ def _catalog_com(job, root): _probe_add_dialog(app, job, desktop, owned_pids, root, library=library) library.Close() library = None + _require_management_continuation(job, root) + update_worker_state(job, 'NATIVE_CALL') library = app.OpenLibrary(str(absolute_preserving_drive(job['library']))) if (library is None or Path(str(library.FullName)).resolve() != Path(job['library']).resolve() or int(library.FlowType) != 1): @@ -1138,6 +1200,20 @@ def management_worker_main(request_path): _, desktop = _current_desktop() if desktop != job['desktop'] or not desktop.startswith('CbbManagement_'): raise RuntimeError('原生管理操作仅允许在本次私有桌面执行。') + _require_management_continuation(job, root) + if job.get('cancel_signal'): + from .desktop_worker import start_supervisor_watchdog + start_supervisor_watchdog(job['supervisor_pid'], cancel_path=job['cancel_signal'], + supervisor_identity=job.get('supervisor_identity'), + session_record_path=job.get('session_record_path'), operation_id=job.get('operation_id')) + if job.get('start_gate'): + deadline = time.monotonic() + 30 + while not Path(job['start_gate']).is_file(): + _require_management_continuation(job, root) + if time.monotonic() >= deadline: + raise RuntimeError('MANAGEMENT_CANCELLED: 启动门未放行,未启动原生操作') + time.sleep(.05) + _require_management_continuation(job, root) if job.get('mapped_drive') and job.get('mapped_remote_root'): from .network_path import ensure_mapped_drive_visible ensure_mapped_drive_visible(job['mapped_drive'], job['mapped_remote_root']) @@ -1165,6 +1241,9 @@ def management_worker_main(request_path): command = [str(adapter.home/'lm/win32/bin/CleanWGLib.exe'), str(job['library']), 'F4', str(input_path)] _write_json(root/'command.json', command) with (root/'native-output.txt').open('wb') as output: + _require_management_continuation(job, root) + from .session_record import update_worker_state + update_worker_state(job, 'NATIVE_CALL') result['safe_to_continue'] = False try: process = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT, @@ -1172,17 +1251,27 @@ def management_worker_main(request_path): except (FileNotFoundError, PermissionError): result['safe_to_continue'] = True raise - _write_json(root/'native-process.json', {'pid': process.pid, 'command': command}) + update_worker_state(job, 'NATIVE_CALL', identities=[('native', process.pid)]) + try: + _write_json(root/'native-process.json', {'pid': process.pid, 'command': command}) + except OSError: + pass # Keep supervising the native owner even when evidence storage fails. # Never terminate this native database owner on timeout. try: code = process.wait(timeout=float(job['timeout_seconds'])) except subprocess.TimeoutExpired: result.update(detail=f'TIMEOUT: CleanWGLib PID {process.pid} 尚未释放') - else: - result.update(safe_to_continue=True, exit_code=code) - text = (root/'native-output.txt').read_bytes().decode('mbcs', errors='replace') - result.update(success=removal_output_ok(code, text, job['name'], job['partition']), - detail=text) + while True: + try: + code = process.wait(timeout=1) + break + except subprocess.TimeoutExpired: + continue + result['wait_exceeded'] = True + result.update(safe_to_continue=True, exit_code=code) + text = (root/'native-output.txt').read_bytes().decode('mbcs', errors='replace') + result.update(success=removal_output_ok(code, text, job['name'], job['partition']), + detail=text) else: raise ValueError('未验证的原生管理操作') except Exception as error: @@ -1191,5 +1280,7 @@ def management_worker_main(request_path): result['detail'] = str(error) result['exception'] = _exception_details(error) finally: + from .session_record import update_worker_state + update_worker_state(job, 'FINISHED' if result.get('safe_to_continue') is True else 'UNCONFIRMED') _write_json(root/'result.json', result) return 0 if result['success'] and result['safe_to_continue'] else 2 diff --git a/tests/test_management.py b/tests/test_management.py index 6c79696..b7cddd5 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -241,7 +241,8 @@ def test_timeout_stops_batch_and_persists_ambiguous_native_stage(setup): adapter.remove = timeout results = run_management_batch([request, replace(request, name='Other')], adapter=adapter, idle_check=lambda *_: None) - assert len(results) == 1 and not results[0].safe_to_continue + assert len(results) == 2 and not results[0].safe_to_continue + assert results[1].status == 'SKIPPED' assert results[0].status == 'RECOVERY_REQUIRED' diff --git a/tests/test_management_diagnostics.py b/tests/test_management_diagnostics.py index f8d9069..d1d6909 100644 --- a/tests/test_management_diagnostics.py +++ b/tests/test_management_diagnostics.py @@ -99,7 +99,8 @@ def test_probe_preloads_uia_before_com_proxy_creation_but_catalog_stays_read_onl monkeypatch.setattr(native, '_probe_add_dialog', editor) monkeypatch.setattr(native, '_wait_native_exit', lambda pids: True) result = native._catalog_native({'operation': operation, 'desktop': 'CbbManagement_test', - 'library': str(library_path), 'mentor_home': str(tmp_path), 'probe_token': 'fixture'}, root) + 'library': str(library_path), 'mentor_home': str(tmp_path), 'probe_token': 'fixture', + 'supervisor_pid': __import__('os').getpid()}, root) assert events[:2] == (['dispatch', 'library-close'] if operation == 'catalog' else ['uia-loaded', 'dispatch']) if operation == 'catalog': assert result['success'] and 'uia-loaded' not in events diff --git a/tests/test_management_probe_guards.py b/tests/test_management_probe_guards.py index 96cf4a0..f5a6a35 100644 --- a/tests/test_management_probe_guards.py +++ b/tests/test_management_probe_guards.py @@ -156,7 +156,8 @@ def test_removal_cli_keeps_mapped_drive_spelling(tmp_path, monkeypatch): from cbb_refresh_verify.management import Capability job = {'operation': 'remove', 'library': 'S:\\library\\mentor.lmc', 'name': 'Block', 'partition': 'Logical-Physical', 'desktop': 'CbbManagement_fixture', - 'mentor_home': str(tmp_path), 'timeout_seconds': 10} + 'mentor_home': str(tmp_path), 'timeout_seconds': 10, + 'supervisor_pid': __import__('os').getpid()} request = tmp_path/'request.json'; request.write_text(json.dumps(job), encoding='utf-8') monkeypatch.setattr(native, '_current_desktop', lambda: (None, job['desktop'])) monkeypatch.setattr(native.NativeManagementAdapter, 'capabilities', lambda self: {'remove': Capability(True, 'fake transport')}) diff --git a/tests/test_management_retry_blockers.py b/tests/test_management_retry_blockers.py index 6431884..ddce289 100644 --- a/tests/test_management_retry_blockers.py +++ b/tests/test_management_retry_blockers.py @@ -179,9 +179,15 @@ def test_batch_cache_binds_exact_library_identity(tmp_path): assert fingerprint_library(first).sha256 != fingerprint_library(second).sha256 -def test_timed_out_management_worker_releases_only_after_its_result(tmp_path, monkeypatch): +def test_timed_out_management_worker_requires_independent_release_proof(tmp_path, monkeypatch): import cbb_refresh_verify.native_management as native state = {'exit': None} + proven = {'released': False} + def release(*a, **kw): + if not proven['released']: + raise OSError('native resources still owned') + return {'status': 'RELEASE_CONFIRMED', 'library': str(library)} + monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', release) worker = SimpleNamespace(poll=lambda: state['exit'], close=lambda: None) adapter = native.NativeManagementAdapter(mentor_home=tmp_path, worker_factory=lambda *_: worker) library = tmp_path/'lib.lmc'; library.touch() @@ -195,6 +201,8 @@ def test_timed_out_management_worker_releases_only_after_its_result(tmp_path, mo assert adapter.release_confirmed(library) is False (evidence/'result.json').write_text(json.dumps({'success': False, 'safe_to_continue': True})) assert adapter.release_confirmed(tmp_path/'other.lmc') is False + assert adapter.release_confirmed(library) is False + proven['released'] = True assert adapter.release_confirmed(library) is True assert worker not in native._retained_workers finally: @@ -483,19 +491,21 @@ def test_native_add_passive_postcheck_failure_preserves_release(transaction, mon @pytest.mark.parametrize('case,safe', [('exit_output_error', True), ('popen_missing', True), - ('popen_denied', True), ('popen_unknown', False), ('timeout', False)]) + ('popen_denied', True), ('popen_unknown', False), ('timeout', True)]) def test_remove_worker_preserves_exit_and_known_creation_rejection(tmp_path, monkeypatch, case, safe): import cbb_refresh_verify.native_management as native job = tmp_path/'request.json' job.write_text(json.dumps({'operation': 'remove', 'desktop': 'CbbManagement_test', 'mentor_home': str(tmp_path), 'library': str(tmp_path/'lib.lmc'), 'name': 'Block', - 'partition': 'Logical-Physical', 'timeout_seconds': 1}), encoding='utf-8') + 'partition': 'Logical-Physical', 'timeout_seconds': 1, + 'supervisor_pid': __import__('os').getpid()}), encoding='utf-8') monkeypatch.setattr(native, '_current_desktop', lambda: (None, 'CbbManagement_test')) monkeypatch.setattr(native.NativeManagementAdapter, 'capabilities', lambda _: { 'remove': SimpleNamespace(supported=True)}) observed = {'waited': False} def wait(timeout): - if case == 'timeout': + if case == 'timeout' and not observed.get('timed_out'): + observed['timed_out'] = True raise native.subprocess.TimeoutExpired('fake CleanWGLib', timeout) observed['waited'] = True return 0 diff --git a/tests/test_management_supervision.py b/tests/test_management_supervision.py new file mode 100644 index 0000000..f2a0c70 --- /dev/null +++ b/tests/test_management_supervision.py @@ -0,0 +1,146 @@ +from dataclasses import replace +import json +import os +from pathlib import Path +from threading import Event + +import pytest + +from cbb_refresh_verify import native_management as native +from cbb_refresh_verify.session_record import process_identity +from test_management import setup, execute + + +@pytest.mark.parametrize('operation', ['catalog', 'add', 'remove']) +@pytest.mark.parametrize('reason', ['cancelled', 'parent_gone', 'identity_changed']) +def test_management_worker_checks_stop_before_any_native_route(tmp_path, monkeypatch, operation, reason): + cancel = tmp_path/'cancel.signal' + identity = process_identity(os.getpid()) + if reason == 'cancelled': + cancel.touch() + if reason == 'identity_changed': + identity['created_handle'] = 'old' + job = dict(operation=operation, library=str(tmp_path/'lib.lmc'), desktop='CbbManagement_Test', + supervisor_pid=-1 if reason == 'parent_gone' else os.getpid(), + supervisor_identity=identity, cancel_signal=str(cancel), operation_id='test-management') + request = tmp_path/'request.json' + request.write_text(json.dumps(job)) + monkeypatch.setattr(native, '_current_desktop', lambda: (None, job['desktop'])) + monkeypatch.setattr(native, '_catalog_native', lambda *a: pytest.fail('unsupervised native route')) + monkeypatch.setattr(native.subprocess, 'Popen', lambda *a, **kw: pytest.fail('unsupervised remove')) + assert native.management_worker_main(request) == 2 + result = json.loads((tmp_path/'result.json').read_text()) + assert not result['success'] + assert result['safe_to_continue'] is True + assert 'CANCELLED' in result['detail'] or '监督器' in result['detail'] + + +def test_cancel_after_remove_stops_update_before_catalog_and_add(setup): + req, adapter, source = setup + stop = Event() + remove = adapter.remove + def stop_after_remove(*args): + result = remove(*args) + stop.set() + return result + adapter.remove = stop_after_remove + result = execute(replace(req, operation='update', source=source), adapter, stop=stop) + assert adapter.calls == ['remove'] + assert result.status == 'RECOVERY_REQUIRED' + assert result.safe_to_continue is True + checkpoint = json.loads(result.evidence_path.read_text('utf-8')) + assert any(item['stage'] == 'native_remove' for item in checkpoint['history']) + + +def test_cancel_before_catalog_does_not_launch_native(setup): + req, adapter, _ = setup + stop = Event() + stop.set() + adapter.catalog = lambda *a: pytest.fail('catalog after cancellation') + result = execute(req, adapter, stop=stop) + assert result.status == 'CANCELLED' + assert result.safe_to_continue is True + + +def test_real_management_worker_parent_loss_waits_for_native_boundary(tmp_path): + import subprocess + import sys + import time + from cbb_refresh_verify.session_record import write_session_record + supervisor = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(30)']) + request, script = tmp_path/'request.json', tmp_path/'worker.py' + gate, cancel, started, continued = [tmp_path/name for name in ('gate', 'cancel', 'started', 'continued')] + job = dict(operation='catalog', library=str(tmp_path/'lib.lmc'), desktop='CbbManagement_test', + supervisor_pid=supervisor.pid, supervisor_identity=process_identity(supervisor.pid), + cancel_signal=str(cancel), start_gate=str(gate), operation_id='real-management', + session_record_path=str(tmp_path/'session.json')) + request.write_text(json.dumps(job)) + script.write_text('import time,sys\nfrom pathlib import Path\n' + 'from cbb_refresh_verify import native_management as n\n' + 'n._current_desktop=lambda:(None,"CbbManagement_test")\n' + 'def native_call(job,root):\n' + f' Path({str(started)!r}).touch()\n' + ' time.sleep(1.5)\n' + ' try: n._require_management_continuation(job,root)\n' + ' except RuntimeError as e: return dict(success=False,safe_to_continue=True,detail=str(e))\n' + f' Path({str(continued)!r}).touch()\n' + ' return dict(success=True,safe_to_continue=True)\n' + 'n._catalog_native=native_call\n' + f'sys.exit(n.management_worker_main({str(request)!r}))\n') + worker = subprocess.Popen([sys.executable, str(script)]) + try: + write_session_record(tmp_path/'session.json', dict(operation_id='real-management', + library=job['library'], processes=[{'role': 'worker', **process_identity(worker.pid)}])) + gate.touch() + deadline = time.monotonic()+8 + while not started.exists() and time.monotonic()