From ddff6bd857239cf97be729c5d9f3c216475cf870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Mon, 16 Nov 2026 18:57:36 +0800 Subject: [PATCH] fix: bound task handoff and preserve native writer ownership --- src/cbb_refresh_verify/application.py | 12 +- src/cbb_refresh_verify/desktop_worker.py | 115 ++- src/cbb_refresh_verify/failed_session.py | 27 + src/cbb_refresh_verify/library_lock.py | 51 +- src/cbb_refresh_verify/management.py | 67 +- src/cbb_refresh_verify/native_management.py | 2 + src/cbb_refresh_verify/native_repair.py | 30 +- src/cbb_refresh_verify/native_repair_ui.py | 2 + src/cbb_refresh_verify/operation_lifecycle.py | 22 +- src/cbb_refresh_verify/operation_queue.py | 302 +++++++- src/cbb_refresh_verify/session_record.py | 2 +- src/cbb_refresh_verify/verify_runner.py | 141 +++- tests/test_batch_item_yield.py | 4 +- tests/test_issue15_task_lifecycle.py | 724 ++++++++++++++++++ tests/test_late_release_continuation.py | 4 +- tests/test_management.py | 2 +- tests/test_native_management.py | 4 + tests/test_review414_write_admission.py | 4 +- 18 files changed, 1397 insertions(+), 118 deletions(-) create mode 100644 tests/test_issue15_task_lifecycle.py diff --git a/src/cbb_refresh_verify/application.py b/src/cbb_refresh_verify/application.py index e85afa5..beaeaf9 100644 --- a/src/cbb_refresh_verify/application.py +++ b/src/cbb_refresh_verify/application.py @@ -98,13 +98,16 @@ def main(argv: Sequence[str] | None = None) -> int: return owner_probe_main(args.owner_probe) if args.native_backup_worker: from .native_backup import native_backup_worker_main - return native_backup_worker_main(args.native_backup_worker) + from .desktop_worker import run_native_worker + return run_native_worker(native_backup_worker_main, args.native_backup_worker) if args.management_worker: from .native_management import management_worker_main - return management_worker_main(args.management_worker) + from .desktop_worker import run_native_worker + return run_native_worker(management_worker_main, args.management_worker) if args.repair_worker: from .native_repair_ui import repair_worker_main - return repair_worker_main(args.repair_worker) + from .desktop_worker import run_native_worker + return run_native_worker(repair_worker_main, args.repair_worker) if args.collect_failure_logs: from .verify_diagnostics import failure_logs_main return failure_logs_main(args.collect_failure_logs) @@ -116,7 +119,8 @@ def main(argv: Sequence[str] | None = None) -> int: import contextlib with args.verify_worker.with_name('worker.log').open('a', encoding='utf-8') as log: with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log): - return worker_main(args.verify_worker) + from .desktop_worker import run_native_worker + return run_native_worker(worker_main, args.verify_worker) context = configure(resolve_data_root(args.data_root)); logger = context.logger; logger.info("application_start") try: if args.batch_lmc: diff --git a/src/cbb_refresh_verify/desktop_worker.py b/src/cbb_refresh_verify/desktop_worker.py index 0da9122..4b3ef81 100644 --- a/src/cbb_refresh_verify/desktop_worker.py +++ b/src/cbb_refresh_verify/desktop_worker.py @@ -16,6 +16,110 @@ _automation_clients = [] _automation_clients_lock = threading.Lock() _supervisor_lost = threading.Event() _native_phases = {} +WRITER_MUTEX = r'Local\CbbRefreshVerify795NativeWriter' + + +def assert_native_writer_idle(): + mutex = win32event.CreateMutex(None, False, WRITER_MUTEX) + acquired = False + try: + acquired = win32event.WaitForSingleObject(mutex, 0) in ( + win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) + if not acquired: + raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回,未启动新的写入。') + finally: + if acquired: win32event.ReleaseMutex(mutex) + mutex.Close() + + +class _RegisteredWriter: + def __init__(self, handle): + import win32api + current = win32api.GetCurrentProcess() + self.handle = win32api.DuplicateHandle(current, handle, current, 0, False, + win32con.DUPLICATE_SAME_ACCESS) + self.returncode = None + + def poll(self): + if self.returncode is None and win32event.WaitForSingleObject(self.handle, 0) == win32con.WAIT_OBJECT_0: + self.returncode = win32process.GetExitCodeProcess(self.handle) + self.handle.Close() + return self.returncode + + +def register_native_writer(process): + handle = getattr(process, 'handle', None) or getattr(process, '_handle', None) + registered = _RegisteredWriter(handle) if handle is not None else process + with _automation_clients_lock: + _automation_clients.append(registered) + return process + + +@contextmanager +def worker_native_channel(*, lease_handle=None, inspection_only=False): + """The native worker, not its disposable supervisor, owns write exclusion.""" + import win32api + mutex = win32event.CreateMutex(None, False, WRITER_MUTEX) + acquired = False + try: + if inspection_only is not True: + acquired = win32event.WaitForSingleObject(mutex, 0) in ( + win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) + if not acquired: + raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回。') + yield + finally: + if acquired: + # Only registered Save/Close/Repair clients extend the boundary; + # an idle PCB shell or a business failure is not a pending write. + warned = False + while True: + with _automation_clients_lock: + clients = tuple(_automation_clients) + try: + released = all(client.poll() is not None for client in clients) + except Exception: + released = False + if not warned: + logging.getLogger(__name__).warning('Native writer exit is unconfirmed', exc_info=True) + warned = True + if released: + break + threading.Event().wait(.1) + win32event.ReleaseMutex(mutex) + mutex.Close() + if lease_handle is not None: + win32api.CloseHandle(int(lease_handle)) + + +def run_native_worker(action, request_path): + """Wait for durable parent admission before acquiring the worker barrier.""" + import json + import time + job = json.loads(Path(request_path).read_text(encoding='utf-8')) + gate = Path(job['start_gate']) if job.get('start_gate') else None + if gate is not None: + deadline = time.monotonic()+30 + while not gate.is_file(): + if (job.get('cancel_signal') and Path(job['cancel_signal']).exists() + or time.monotonic() >= deadline): + raise RuntimeError('WORKER_CANCELLED: 未取得启动许可。') + threading.Event().wait(.05) + lease_handle = None + if job.get('session_record_path'): + from .session_record import read_session_record + path = Path(job['session_record_path']) + record = read_session_record(path, evidence_root=path.parent) + if (record['operation_id'] != job['operation_id'] + or not any(p['pid'] == os.getpid() for p in record.get('processes', ()) )): + raise RuntimeError('WORKER_ADMISSION_IDENTITY_CHANGED') + lease_handle = record.get('worker_lease_handle') + inspection = (action.__module__ == 'cbb_refresh_verify.native_repair_ui' + and action.__name__ == 'repair_worker_main' and job.get('inspection_only') is True) + with worker_native_channel(lease_handle=lease_handle, inspection_only=inspection): + if job.get('cancel_signal') and Path(job['cancel_signal']).exists(): + raise RuntimeError('WORKER_CANCELLED: 已取消原生启动。') + return action(request_path) def _phase_pending(phase): @@ -56,6 +160,8 @@ def _record_write_pending(record_path, operation_id=None): def launch_save_helper(command, *, start_gate=None, identity_path=None, operation_id=None, **kwargs): """Register the save client atomically with respect to supervisor loss.""" with _automation_clients_lock: + from .operation_queue import check_active + check_active() if _supervisor_lost.is_set(): raise RuntimeError('VERIFY_CANCELLED: 监督器已退出,未启动新的保存客户端') helper = subprocess.Popen(command, **kwargs) @@ -160,6 +266,8 @@ class DesktopWorker: startup = win32process.STARTUPINFO() # subprocess.STARTUPINFO does NOT pass lpDesktop to CreateProcess. startup.lpDesktop = startup_desktop_name(name, interactive_input=self.interactive_input) + from .operation_queue import check_active + check_active() self.handle, thread, self.pid, _ = win32process.CreateProcess( command[0], subprocess.list2cmdline(command), None, None, False, win32con.CREATE_NO_WINDOW, None, None, startup) @@ -199,6 +307,8 @@ class DesktopWorker: raise def start_companion(self, command): + from .operation_queue import check_active + check_active() startup = win32process.STARTUPINFO() startup.lpDesktop = startup_desktop_name( self.name, interactive_input=getattr(self, 'interactive_input', False)) @@ -283,7 +393,10 @@ class DesktopWorker: def bind_session(self, job): from .session_record import register_worker_session - register_worker_session(self, job) + from .library_lock import duplicate_worker_lease + lease = (None if job.get('inspection_only') is True else + duplicate_worker_lease(job['library'], self.handle)) + register_worker_session(self, {**job, 'worker_lease_handle': lease}) def native_write_pending(self): return _record_write_pending(getattr(self, 'session_record_path', None)) diff --git a/src/cbb_refresh_verify/failed_session.py b/src/cbb_refresh_verify/failed_session.py index 421342d..a969768 100644 --- a/src/cbb_refresh_verify/failed_session.py +++ b/src/cbb_refresh_verify/failed_session.py @@ -53,6 +53,14 @@ def _cancel_target(snapshot): return None +def _tip_target(snapshot): + from .dialog_agent import DialogActionExecutor + if snapshot.title != 'Tip of the Day' or snapshot.window_class != '#32770': + return None + target = DialogActionExecutor._target(snapshot, {'control_id': 1}) + return target if target is not None and target.class_name == 'Button' else None + + def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: None, timeout_seconds=20, launcher_process=None): """Normally close our empty PCB after a diagnosed database startup failure. @@ -84,6 +92,7 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: declined_hwnd = None declined_kind = None cancelled_open = False + dismissed_tips = set() while True: records = [r for r in _process_details('ExpeditionPCB.exe') if r[0] == pcb_pid] if not records: @@ -121,6 +130,24 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: if declined_hwnd is not None and declined_hwnd not in windows: cancelled_open = True result['cancelled_prompt'] = declined_kind + dismissed = False + for hwnd in windows: + if hwnd in dismissed_tips or win32gui.GetClassName(hwnd) != '#32770': + continue + snapshot = _startup_snapshot(hwnd, pcb_pid) + target = _tip_target(snapshot) + if target is None: + continue + current = _startup_snapshot(hwnd, pcb_pid) + if current != snapshot or _tip_target(current) != target: + raise ValueError('STARTUP_TIP_CHANGED') + win32gui.PostMessage(target.handle, win32con.BM_CLICK, 0, 0) + dismissed_tips.add(hwnd) + dismissed = True + emit(stage='startup_tip_dismissed', pcb_pid=pcb_pid, hwnd=hwnd) + break + if dismissed: + continue # Prove the Tip disappeared; it does not prove a cancelled open. if declined_hwnd is None: for hwnd in windows: if win32gui.GetClassName(hwnd) != '#32770': diff --git a/src/cbb_refresh_verify/library_lock.py b/src/cbb_refresh_verify/library_lock.py index 9e0b09c..e6141f7 100644 --- a/src/cbb_refresh_verify/library_lock.py +++ b/src/cbb_refresh_verify/library_lock.py @@ -20,6 +20,21 @@ def _library_guard(key): return _library_guards.setdefault(key, threading.RLock()) +def duplicate_worker_lease(library, process_handle): + """Keep the exact admitted exclusive file object alive across parent exit.""" + import win32api, win32con + lease = LibraryLock(library) + with _library_guard(lease.key): + owner = _owners.get(lease.key) + if owner is None: + return None + if owner['thread'] != threading.get_ident() or owner['retained'] or not owner['depth']: + raise LibraryBusy('不能将其他任务的库锁交给工作进程。') + remote = win32api.DuplicateHandle(win32api.GetCurrentProcess(), owner['handle'], + process_handle, 0, False, win32con.DUPLICATE_SAME_ACCESS) + return remote.Detach() # This handle belongs to the target process only. + + class LibraryBusy(OSError): pass @@ -407,14 +422,12 @@ def _owner_is_blocking(item, library, roots): def _enter_maintenance(lease): """Transfer only a valid known lease, after project admission under lock.""" - import win32con, win32event, win32file - mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') - acquired = False + import win32con, win32file + from .operation_queue import check_active handle = None existing = None try: - acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) - if not acquired: raise LibraryBusy('已有 CBB 操作运行,维护不能接续') + check_active() with _library_guard(lease.key): marker = blocker_path(lease.library) if marker.with_suffix('.pending').exists(): @@ -449,6 +462,7 @@ def _enter_maintenance(lease): with saved.open('xb') as stream: stream.write(old); stream.flush(); os.fsync(stream.fileno()) if saved.read_bytes() != old: raise OSError('原检查点归档校验失败') + check_active() lease_id = uuid4().hex _persist_blocker(lease.library, 'OPERATION_IN_PROGRESS', lease_id=lease_id) _owners[lease.key] = {'thread': threading.get_ident(), 'depth': 1, 'handle': handle, @@ -456,12 +470,9 @@ def _enter_maintenance(lease): 'release_probe': lease.safe_to_release, 'block_name': None} lease.maintenance_proof = proof lease.entered = True - lease.maintenance_mutex = mutex return lease except Exception: if handle is not None and existing is None: handle.Close() - if acquired: win32event.ReleaseMutex(mutex) - mutex.Close() raise @@ -550,7 +561,13 @@ def _prove_library_released(library, block_name=None, *, block_names=(), allow_u # and compare the exact address/project binding before commit. stack.close() from .icdb_ownership import inspect_release_addresses - inspections = inspect_release_addresses(files, stop=stop) + # Checkpoint I/O is per-library; only native inspection owns + # the host channel. The sidecar still excludes other writers. + from .operation_queue import native_channel + with native_channel(inspection_only=True): + if stop is not None and stop.is_set(): + raise InterruptedError('已停止释放检查') + inspections = inspect_release_addresses(files, stop=stop) try: current = stack.enter_context(hold_release_files(library, roots, stop=stop)) except OSError as error: @@ -582,7 +599,7 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na The upload sidecar is never deleted. This acknowledges process release, not the correctness of a partially changed library or its recovery state. """ - import win32con, win32event, win32file + import win32con, win32file library = Path(library).resolve(strict=True) from .session_record import (retained_session_snapshot, acknowledge_library_release, reconciled_retained_sessions) @@ -594,15 +611,10 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na if evidence.is_relative_to(library.parent) or library.parent.is_relative_to(evidence): raise ValueError('释放检查证据必须位于目标库之外。') lease = LibraryLock(library) - mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') - acquired = False external = None try: if stop is not None and stop.is_set(): raise InterruptedError('已停止释放检查') - acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) - if not acquired: - raise LibraryBusy('另一个 CBB 操作仍在运行。') with _library_guard(lease.key): owner = _owners.get(lease.key) if owner and (not owner['retained'] or owner['depth']): @@ -675,7 +687,8 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na 'block_names': list(scoped_blocks), 'lease_id': owner['lease_id'] if owner else None, 'operation_ids': [item['operation_id'] for item in reconciled_sessions]} - (root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8') + pending = {**record, 'status': 'RELEASE_PENDING', 'lease_state': 'PENDING'} + (root/'release.json').write_text(json.dumps(pending, ensure_ascii=False, indent=2), encoding='utf-8') for index, checkpoint in enumerate(checkpoints): if checkpoint.exists(): data = checkpoint.read_bytes() @@ -683,15 +696,15 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na archived.write_bytes(data) if archived.read_bytes() != data: raise OSError('阻止记录归档校验失败,保持库锁。') + if stop is not None and stop.is_set(): + raise InterruptedError('已停止释放检查') checkpoint.unlink() # Checkpoint JSON only; never .lock. if owner: owner['handle'].Close() del _owners[lease.key] + (root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8') acknowledge_library_release(record, matching_retained, block_name=scoped) return record finally: if external is not None: external.Close() - if acquired: - win32event.ReleaseMutex(mutex) - mutex.Close() diff --git a/src/cbb_refresh_verify/management.py b/src/cbb_refresh_verify/management.py index 5bf1810..6f646ab 100644 --- a/src/cbb_refresh_verify/management.py +++ b/src/cbb_refresh_verify/management.py @@ -173,7 +173,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro write_admission=None): from .verify_runner import VerifyRequest, run_verify from .backup_recovery import _idle_check - import win32con, win32event + from contextlib import ExitStack + from .operation_queue import check_active, native_channel, task_stop + stop = task_stop(stop) import pythoncom if adapter is None: from .native_management import NativeManagementAdapter @@ -190,8 +192,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro history = [] mutated = uploaded = removed = False owner_safe = True - mutex = None - acquired = False release_proof = None original = None change_report = None @@ -199,12 +199,13 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro nonlocal owner_safe if stop is not None and stop.is_set(): raise InterruptedError('MANAGEMENT_CANCELLED: 已停止后续原生操作') - owner_safe = False - try: - result = callback(*args) - except Exception as error: - owner_safe = getattr(error, 'safe_to_continue', False) is True - raise + with native_channel(): + owner_safe = False + try: + result = callback(*args) + except Exception as error: + owner_safe = getattr(error, 'safe_to_continue', False) is True + raise owner_safe = result.safe_to_continue return result def checkpoint(next_stage, **details): @@ -292,14 +293,10 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro root.mkdir(parents=True, exist_ok=False) report = root/'checkpoint.json' checkpoint('release_check') - mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') - acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) - if not acquired: - return finish('BLOCKED', '已有 CBB 校验/管理/恢复任务运行;本次未启动。') from .library_lock import recheck_previous_release release_proof = recheck_previous_release(library, root/'previous-release', block_name=request.name, required=request.check_previous_release or bool(unfinished_target - and unfinished_target.get('safe_to_continue') is not True), progress=progress) + and unfinished_target.get('safe_to_continue') is not True), progress=progress, stop=stop) if release_proof is not None: checkpoint('release_check', release_proof=release_proof) if unfinished_target: @@ -308,8 +305,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro adapter.bind_library_mapping(request.mapped_drive, request.mapped_remote_root) if hasattr(adapter,'bind_write_admission'): adapter.bind_write_admission(write_admission if native_idle_check else None) + check_active() with LibraryLock(library, block_name=request.name, safe_to_release=lambda: owner_safe or ( - callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))): + callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))), ExitStack() as native_stack: work = library.parent/'ReusableBlocksLP'/request.name/'Work' target_project, target_pcb = work/(request.name+'.prj'), work/'PCB/pcb.pcb' if request.operation != 'add': @@ -324,7 +322,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro if source: idle_check(library, source, pcb) checkpoint('catalog_before') - owner_safe = False before = invoke(adapter.catalog, library, root/'before') owner_safe = before.safe_to_continue if not owner_safe or Path(before.library).resolve() != canonical_library or before.flow != 1: @@ -366,6 +363,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro if request.operation in ('remove', 'update'): if stop is not None and stop.is_set(): return finish('CANCELLED', '已取消,未开始移除', owner_safe) + native_stack.enter_context(native_channel()) checkpoint('native_remove') mutated = True; owner_safe = False native = invoke(adapter.remove, request, root/'remove') @@ -385,6 +383,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro if stop is not None and stop.is_set(): return finish('RECOVERY_REQUIRED' if mutated else 'CANCELLED', '已停止后续导入;已发生的原生变更需要核对', owner_safe) + if not removed: + native_stack.enter_context(native_channel()) checkpoint('native_add') mutated = True; owner_safe = False native = invoke(adapter.add, staged, root/'add') @@ -413,6 +413,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro return finish('RECOVERY_REQUIRED', '原生新增后名称、分区或工程关联未确认一致。\n'+native.detail, owner_safe) uploaded = True checkpoint('uploaded') + native_stack.close() if not request.auto_verify: return finish('UPLOADED', '上传已确认;本批次未启用自动校验。\n'+native.detail) # Adapter catalog cannot return until its exact native owner exits. @@ -442,29 +443,29 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro status = 'BLOCKED' return finish(status, str(error), owner_safe) finally: - if mutex is not None: - if acquired: - win32event.ReleaseMutex(mutex) - mutex.Close() uninitialize_com(initialized) -def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, **kwargs): +def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, context=None, **kwargs): from threading import Event from .write_admission import WriteAdmission + from .operation_queue import QueueCancelled, TaskHandoff, request_library_keys + if context is not None: + stop = stop if stop is not None else context.stop + progress = context.guard_callback(progress) if progress else None if stop is None: stop = Event() kwargs.setdefault('write_admission',WriteAdmission(prompt=kwargs.get('version_prompt'),stop=stop)) results = [] retained_libraries = set() + cancelled = False for request in requests: - if stop is not None and stop.is_set(): - break - key = str(request.library).casefold() - if key in retained_libraries: + keys = request_library_keys(request, context) + stopped = cancelled or stop.is_set() or context is not None and context.stop.is_set() + if stopped or keys.intersection(retained_libraries): result = ManagementResult(request.operation, request.library, request.name, request.partition, request.source, request.auto_verify, 'complete', 'SKIPPED', True, - '同库原生会话尚未释放,未启动新的写入') + '队列已停止,未操作此 CBB' if stopped else '同库原生会话尚未释放,未启动新的写入') results.append(result) if progress: progress({'stage': result.status, 'block': result.name, 'detail': result.detail}) @@ -473,21 +474,21 @@ def run_management_batch(requests, *, stop=None, progress=None, item_executor=No result = run_management(request, progress=progress, stop=stop, **kwargs) if not result.safe_to_continue: from .operation_lifecycle import recheck_failed_release - result = recheck_failed_release(request.library, request.name, result) + result = recheck_failed_release(request.library, request.name, result, stop=stop) return result - from .operation_queue import QueueCancelled try: result = item_executor(request, action) if item_executor is not None else action() except QueueCancelled as error: - results.append(ManagementResult(request.operation, request.library, request.name, request.partition, - request.source, request.auto_verify, 'complete', 'CANCELLED', True, str(error))) - break + result = ManagementResult(request.operation, request.library, request.name, request.partition, + request.source, request.auto_verify, 'complete', + 'TIMEOUT' if isinstance(error, TaskHandoff) and not error.cancelled else 'CANCELLED', + not isinstance(error, TaskHandoff), str(error)) results.append(result) if progress: progress({'stage': result.status, 'block': result.name, 'detail': result.detail, 'evidence_path': str(result.evidence_path or '')}) if not result.safe_to_continue: - retained_libraries.add(key) + retained_libraries.update(keys) if result.status == 'CANCELLED': - break + cancelled = True return results diff --git a/src/cbb_refresh_verify/native_management.py b/src/cbb_refresh_verify/native_management.py index 4235476..1df8ffa 100644 --- a/src/cbb_refresh_verify/native_management.py +++ b/src/cbb_refresh_verify/native_management.py @@ -1285,6 +1285,8 @@ def management_worker_main(request_path): except (FileNotFoundError, PermissionError): result['safe_to_continue'] = True raise + from .desktop_worker import register_native_writer + register_native_writer(process) update_worker_state(job, 'NATIVE_CALL', identities=[('native', process.pid)], continuation=True) try: _write_json(root/'native-process.json', {'pid': process.pid, 'command': command}) diff --git a/src/cbb_refresh_verify/native_repair.py b/src/cbb_refresh_verify/native_repair.py index 3ec1a5f..0552ec2 100644 --- a/src/cbb_refresh_verify/native_repair.py +++ b/src/cbb_refresh_verify/native_repair.py @@ -264,15 +264,34 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr job_path = root/'repair-request.json' cancel = root/'cancel-repair.signal' desktop = 'CbbRepair_'+uuid4().hex[:12] - job_path.write_text(json.dumps({'project': str(target.project), 'database': str(target.database), + from .operation_queue import current_context + context = current_context() + library = (maintenance_context or {}).get('library') or (context.library if context else str(target.project)) + job = {'project': str(target.project), 'database': str(target.database), 'root': str(root), 'mentor_bin': str(mentor_bin or ''), 'timeout': timeout_seconds, 'cancel_signal': str(cancel), 'supervisor_pid': os.getpid(), 'desktop': desktop, - 'maintenance_context': maintenance_context, 'inspection_only': inspection_only}, - ensure_ascii=False), encoding='utf-8') + 'maintenance_context': maintenance_context, 'inspection_only': inspection_only, + 'library': library, 'operation_id': root.name+'-'+uuid4().hex[:8], + 'operation': 'inspect' if inspection_only else 'repair', + 'block': (maintenance_context or {}).get('block'), + 'session_record_path': str(root/'session.json'), 'start_gate': str(root/'start-repair.signal')} + job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8') command = ([sys.executable] if getattr(sys, 'frozen', False) else [sys.executable, '-m', 'cbb_refresh_verify']) + ['--repair-worker', str(job_path)] + from .operation_queue import native_channel, task_stop + with native_channel(inspection_only=inspection_only is True): + return _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, task_stop(stop), job) + + +def _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, stop, job): + from .desktop_worker import DesktopWorker worker = DesktopWorker(command, desktop) worker.bind_cancellation(cancel) + if hasattr(worker, 'bind_session'): + worker.bind_session(job) + from .operation_queue import check_active + check_active() + Path(job['start_gate']).touch() # Worker admission includes filesystem/ownership checks before the native # timeout begins. Bound that preparation separately; never extend on polls. deadline = time.monotonic()+60+max(1, float(timeout_seconds))+5 @@ -307,4 +326,9 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr result['status'] = 'UNCONFIRMED' return {**result, 'evidence_path': str(root)} finally: + if worker.poll() is None and getattr(worker, 'session_record_path', None): + from .session_record import handoff_session + from .operation_queue import hold_native_handoff + handoff_session(worker.session_record_path, evidence_root=root, process=worker) + hold_native_handoff(worker, worker.session_record_path) worker.close() diff --git a/src/cbb_refresh_verify/native_repair_ui.py b/src/cbb_refresh_verify/native_repair_ui.py index 9642297..6e29bd5 100644 --- a/src/cbb_refresh_verify/native_repair_ui.py +++ b/src/cbb_refresh_verify/native_repair_ui.py @@ -335,6 +335,8 @@ def repair_worker_main(job_path): if Path(job['cancel_signal']).exists(): raise ValueError('REPAIR_CANCELLED_BEFORE_LAUNCH') process = subprocess.Popen([str(executable), '-p', str(project)], cwd=str(project.parent)) + from .desktop_worker import register_native_writer + register_native_writer(process) result['native_started'] = True result['native_pid'] = process.pid emit(stage='native_started', pid=process.pid, project=str(project)) diff --git a/src/cbb_refresh_verify/operation_lifecycle.py b/src/cbb_refresh_verify/operation_lifecycle.py index 105e87c..fcdec8d 100644 --- a/src/cbb_refresh_verify/operation_lifecycle.py +++ b/src/cbb_refresh_verify/operation_lifecycle.py @@ -2,10 +2,19 @@ from collections.abc import Sequence -def recheck_failed_release(library, block, result): +def recheck_failed_release(library, block, result, *, stop=None, timeout_seconds=10): """Refresh resource facts after a late exit; keep the original business failure.""" if result.safe_to_continue or not result.evidence_path: return result + from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop + if current_context() is None: + context = BatchTaskContext(library=str(library), targets=(block,)) + try: + return context.run(lambda: recheck_failed_release(library, block, result, stop=stop), + stop=stop, timeout_seconds=timeout_seconds) + except TaskHandoff: + return result + stop = task_stop(stop) from pathlib import Path from dataclasses import replace import json @@ -37,9 +46,16 @@ def recheck_failed_release(library, block, result): bound.append(candidate) if not bound: return result # A generic timeout without a bound owner is not an exit fact. - proof = release_retained_library(library, evidence.parent/'late-release', block_name=block) - if proof.get('status') != 'RELEASE_CONFIRMED' or Path(proof.get('library', '')).resolve() != Path(library).resolve(): + from .operation_queue import check_active + check_active() + proof = release_retained_library(library, evidence.parent/'late-release', block_name=block, stop=stop) + scope = proof.get('resource_scope') + if (proof.get('status') != 'RELEASE_CONFIRMED' or proof.get('lease_state') != 'RELEASED' + or not isinstance(scope, dict) or scope.get('scope_state') != 'OBSERVED' + or proof.get('blocked_resources') != [] + or Path(proof.get('library', '')).resolve() != Path(library).resolve()): return result + check_active() evidence.with_name('late-release-result.json').write_text(json.dumps(dict( business_status=result.status, safe_to_continue=True, release_proof=proof), ensure_ascii=False, default=str, indent=2), 'utf-8') diff --git a/src/cbb_refresh_verify/operation_queue.py b/src/cbb_refresh_verify/operation_queue.py index 95fd7a1..661fe48 100644 --- a/src/cbb_refresh_verify/operation_queue.py +++ b/src/cbb_refresh_verify/operation_queue.py @@ -1,36 +1,320 @@ """Cooperative FIFO admission for this application's native operations.""" -from threading import Condition +from contextlib import contextmanager +from dataclasses import dataclass, field +import ntpath +from threading import Condition, Event, RLock, Thread, local +import time +from uuid import uuid4 class QueueCancelled(RuntimeError): pass +class TaskHandoff(QueueCancelled): + """The call is still observable; this is never a release receipt.""" + def __init__(self, context, *, cancelled=False, result=None): + self.operation_id = context.operation_id + self.library = context.library + self.cancelled = cancelled + self.scope = context.scope + self.result = result + super().__init__('已交给后台观察;当前资源尚未确认释放。') + + +class _StopSignal: + def __init__(self): + self._event = Event() + + def set(self): + self._event.set() + + def is_set(self): + return self._event.is_set() + + def wait(self, timeout=None): + return self._event.wait(timeout) + + +_local = local() +_tasks_guard = RLock() +_calls = {} + + +def _path_key(path): + # Identity was bound by the caller; queue predicates must never resolve a share. + return ntpath.normcase(ntpath.normpath(str(path))) + + +def request_library_keys(request, context=None): + library = str(getattr(request, 'lmc_file', getattr(request, 'library', '__injected_verifier_scope__'))) + keys = {_path_key(library)} + drive, remote = getattr(request, 'mapped_drive', ''), getattr(request, 'mapped_remote_root', '') + if drive and remote and ntpath.splitdrive(library)[0].casefold() == drive.rstrip('\\/').casefold(): + keys.add(_path_key(ntpath.join(remote, ntpath.splitdrive(library)[1].lstrip('\\/')))) + if context is not None: + aliases = {_path_key(v) for v in (context.library, *context.aliases)} + if keys.intersection(aliases): + keys.update(aliases) + return keys + + +@dataclass(eq=False) +class _Call: + context: object + external_stop: object = None + done: Event = field(default_factory=Event) + abandoned: bool = False + native_depth: int = 0 + native_handoff: str = '' + phase_started: float = field(default_factory=time.monotonic) + result: object = None + last_result: object = None + error: BaseException | None = None + + def is_set(self): + return (self.abandoned or self.context.stop.is_set() + or self.external_stop is not None and self.external_stop.is_set()) + + +def current_context(): + call = getattr(_local, 'call', None) + return call.context if call is not None else None + + +def task_stop(fallback=None): + return getattr(_local, 'call', None) or fallback + + +def check_active(): + call = getattr(_local, 'call', None) + if call is not None and call.is_set(): + raise QueueCancelled('任务已取消,未提交新的原生操作。') + + +def remember_result(result): + call = getattr(_local, 'call', None) + if call is not None: + with _tasks_guard: + call.last_result = result + return result + + +def hold_native_handoff(process, session_record_path): + """Keep this thread's mutex while a recorded native client is unreturned. + + The caller registers the observer first. Only UI/scheduling is handed off; + this thread and its native channel remain alive until that exact handle exits. + """ + call = getattr(_local, 'call', None) + if call is not None: + with _tasks_guard: + if not call.native_depth: + raise RuntimeError('Native handoff requires an owned native phase') + call.native_handoff = str(session_record_path) + while process.poll() is None: + Event().wait(.1) + + +@contextmanager +def native_phase(): + """Never abandon a transaction between native writes (including Remove/Add).""" + call = getattr(_local, 'call', None) + with _tasks_guard: + check_active() + if call is not None: + call.native_depth += 1 + try: + yield + finally: + if call is not None: + with _tasks_guard: + call.native_depth -= 1 + call.phase_started = time.monotonic() + + +@contextmanager +def native_channel(*, inspection_only=False): + """Shared by Verify, Repair and backup native clients; reentrant per thread.""" + import win32con, win32event + with native_phase(): + mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') + acquired = False + try: + check_active() + acquired = win32event.WaitForSingleObject(mutex, 0) in ( + win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) + if not acquired: + raise RuntimeError('NATIVE_CHANNEL_BUSY: 另一个原生操作仍在运行。') + from .desktop_worker import assert_native_writer_idle + if inspection_only is not True: + assert_native_writer_idle() + yield + finally: + if acquired: + win32event.ReleaseMutex(mutex) + mutex.Close() + + +@dataclass(frozen=True) +class BatchTaskContext: + library: str + aliases: tuple[str, ...] = () + targets: tuple[str, ...] = () + resources: tuple[str, ...] = () + operation_id: str = field(default_factory=lambda: uuid4().hex) + stop: _StopSignal = field(default_factory=_StopSignal, repr=False, compare=False) + _closed: Event = field(default_factory=Event, repr=False, compare=False) + + def __post_init__(self): + object.__setattr__(self, 'library', str(self.library)) + for name in ('aliases', 'targets', 'resources'): + object.__setattr__(self, name, tuple(str(v) for v in getattr(self, name))) + + @property + def scope(self): + return dict(scope_state='DECLARED', library=self.library, aliases=list(self.aliases), + block_names=list(self.targets), resources=list(self.resources), + operation_ids=[self.operation_id]) + + @property + def state(self): + return self.snapshot()['state'] + + def snapshot(self): + with _tasks_guard: + calls = [call for call in _calls.values() if call.context is self] + pending = sum(not call.done.is_set() for call in calls) + return dict(operation_id=self.operation_id, resource_scope=self.scope, + state='CLOSED' if self._closed.is_set() else + 'CANCELLED' if self.stop.is_set() else + 'HANDED_OFF' if any(c.abandoned for c in calls) else + 'RUNNING' if pending else 'IDLE', pending=pending, + pending_native=any(c.native_depth for c in calls)) + + def cancel(self): + self.stop.set() + + def check_active(self): + if self.stop.is_set() or self._closed.is_set(): + raise QueueCancelled('任务已取消,未提交新的原生操作。') + check_active() + + def guard_callback(self, callback, *, current=None): + call = getattr(_local, 'call', None) + def guarded(*args, **kwargs): + # Cancellation still gets its own terminal result; closing or an + # abandoned invocation cannot update a newer task/library. + if (not self._closed.is_set() and not (call and call.abandoned) + and (current is None or current() is self)): + return callback(*args, **kwargs) + return guarded + + def close(self, *, timeout_seconds=30): + self._closed.set() + self.cancel() + deadline = time.monotonic() + min(30, max(0, timeout_seconds)) + with _tasks_guard: + calls = [c for c in _calls.values() if c.context is self] + for call in calls: + call.done.wait(max(0, deadline-time.monotonic())) + with _tasks_guard: + for call in calls: + if not call.done.is_set() and not call.native_depth: + call.abandoned = True + return self.snapshot() + + def run(self, action, *, stop=None, timeout_seconds=10): + self.check_active() + if current_context() is self: + return action() # Preserve nested lease and mutex thread identity. + aliases = {_path_key(v) for v in (self.library, *self.aliases)} + call = _Call(self, stop) + with _tasks_guard: + if any(c.abandoned and aliases.intersection( + _path_key(v) for v in (c.context.library, *c.context.aliases)) + for c in _calls.values()): + raise TaskHandoff(self) + _calls[id(call)] = call + def execute(): + _local.call = call + try: + check_active() + call.result = action() + except BaseException as error: + call.error = error + finally: + with _tasks_guard: + call.done.set() + _calls.pop(id(call), None) + del _local.call + Thread(target=execute, name='cbb-task-'+self.operation_id[:12], daemon=True).start() + while not call.done.wait(.02): + with _tasks_guard: + if call.native_handoff: + cancelled = call.is_set() + call.abandoned = True + raise TaskHandoff(self, cancelled=cancelled, result=call.last_result) + elapsed = time.monotonic()-call.phase_started + if not call.native_depth and (call.abandoned or elapsed >= timeout_seconds): + cancelled = call.is_set() + call.abandoned = True + raise TaskHandoff(self, cancelled=cancelled, result=call.last_result) + if call.error is not None: + raise call.error + return call.result + + class NativeQueue: def __init__(self): self.condition = Condition() self.pending = [] self.active = False + self.closed = False + self.contexts = {} def wake(self): with self.condition: self.condition.notify_all() - def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None): - ticket = object() + def close(self, *, timeout_seconds=30): + deadline = time.monotonic()+min(30, max(0, timeout_seconds)) + with self.condition: + self.closed = True + contexts = tuple(self.contexts.values()) + for context in contexts: + context.cancel() + self.condition.notify_all() + return tuple(context.close(timeout_seconds=max(0, deadline-time.monotonic())) + for context in contexts) + + def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None, + context=None, timeout_seconds=10): + ticket = [object(), False] acquired, notified = False, False previous_state = None - with self.condition: self.pending.append((ticket, ready)) + with self.condition: + if self.closed: + raise QueueCancelled('队列已关闭,未启动新的操作。') + if context is not None: + self.contexts[id(context)] = context + self.pending.append(ticket) try: while not acquired: + available = ready() # Call only this ticket's predicate, outside the queue lock. with self.condition: - if stop is not None and stop.is_set(): + if (self.closed or stop is not None and stop.is_set() + or context is not None and context.stop.is_set()): raise QueueCancelled('已取消排队,未启动原生操作。') - first = next((token for token, available in self.pending if available()), None) + ticket[1] = bool(available) + first = next((entry for entry in self.pending if entry[1]), None) if not self.active and first is ticket: - self.pending = [entry for entry in self.pending if entry[0] is not ticket] self.active = acquired = True else: self.condition.wait(.1) + if acquired and not ready(): + with self.condition: + ticket[1] = self.active = acquired = False + self.condition.notify_all() if not acquired and waiting is not None: try: current_state = wait_state() if wait_state is not None else None @@ -41,9 +325,9 @@ class NativeQueue: pass if stop is not None and stop.is_set(): raise QueueCancelled('已取消排队,未启动原生操作。') - return action() + return context.run(action, stop=stop, timeout_seconds=timeout_seconds) if context else action() finally: with self.condition: - self.pending = [entry for entry in self.pending if entry[0] is not ticket] + self.pending = [entry for entry in self.pending if entry is not ticket] if acquired: self.active = False self.condition.notify_all() diff --git a/src/cbb_refresh_verify/session_record.py b/src/cbb_refresh_verify/session_record.py index 1414122..0bcc19a 100644 --- a/src/cbb_refresh_verify/session_record.py +++ b/src/cbb_refresh_verify/session_record.py @@ -130,7 +130,7 @@ def register_worker_session(worker, job): path = Path(job['session_record_path']) from .icdb_ownership import session_resource_scope write_session_record(path, {k: job.get(k) for k in ( - 'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal')} + 'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal', 'worker_lease_handle')} | {'processes': [{'role': 'worker', **process_identity(worker.pid)}], 'lease_state': 'ACTIVE', 'resource_scope': session_resource_scope(job), 'blocked_resources': [], 'data_recovery_required': None}) diff --git a/src/cbb_refresh_verify/verify_runner.py b/src/cbb_refresh_verify/verify_runner.py index 0a63eae..1a76cb6 100644 --- a/src/cbb_refresh_verify/verify_runner.py +++ b/src/cbb_refresh_verify/verify_runner.py @@ -545,14 +545,57 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, l def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, - dialog_supervisor_factory=None, version_prompt=None, stop=None): - # Cover contents checking, native work and final state reads with one - # session-wide mutex. The worker-stage lock below is reentrant on Windows. - import win32event, win32con + dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None, + admission_timeout_seconds=10): + from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop + options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner, + port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory, + version_prompt=version_prompt) + bound = current_context() + if bound is None: + # Management may call Verify while holding a same-thread lease. Do not + # transfer that transaction to a different thread. + import threading + from .library_lock import _guard, _owners + with _guard: + nested = any(v['thread'] == threading.get_ident() and v['depth'] > 0 + for v in _owners.values()) + if not nested: + from .operation_queue import request_library_keys + bound = context or BatchTaskContext(library=str(request.lmc_file), + aliases=tuple(sorted(request_library_keys(request))), targets=(request.block_name,)) + try: + return bound.run(lambda: run_verify(request, stop=stop, **options), + stop=stop, timeout_seconds=admission_timeout_seconds) + except TaskHandoff as error: + result = _handoff_verify_result(error) + _notify_progress(bound.guard_callback(progress) if progress else None, + dict(stage='task_handoff', operation_id=bound.operation_id, + library=bound.library, status=result.status, resource_scope=error.scope)) + return result + if bound is not None and progress is not None: + options['progress'] = bound.guard_callback(progress) + return _run_verify_task(request, stop=task_stop(stop), **options) + + +def _handoff_verify_result(error): + known = error.result if isinstance(error.result, VerifyResult) else None + result = known or VerifyResult('CANCELLED' if error.cancelled else 'TIMEOUT') + return replace(result, status='FAIL' if result.status == 'PASS' else result.status, + stderr=result.stderr or str(error), safe_to_continue=False, + error_stage=result.error_stage or 'admission_or_finalization', + error_category=result.error_category or 'TASK_HANDOFF', + release_proof=dict(lease_state='PENDING', resource_scope=error.scope, + blocked_resources=error.scope.get('resources', []))) + + +def _run_verify_task(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, + dialog_supervisor_factory=None, version_prompt=None, stop=None): + # Admission/finalization keep only the library lease. Native launch paths + # acquire the host channel after slow metadata has returned. + from .operation_queue import check_active from .com_runtime import initialize_com, uninitialize_com initialized = initialize_com() - mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') - acquired = False library_lock = None previous_release = None release_confirmed = True @@ -563,11 +606,7 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, from .library_lock import _supervisor_alive return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients) try: - acquired = win32event.WaitForSingleObject(mutex, 0) in ( - win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) - if not acquired: - return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。', - safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY') + check_active() if request.maintenance: from .maintenance_entry import run_icdb_check return run_icdb_check(request, progress=progress, stop=stop) @@ -576,10 +615,13 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, if Path(request.lmc_file).is_file(): try: previous_release = recheck_previous_release(request.lmc_file, - Path(request.evidence_dir)/'previous-release', block_name=request.block_name, progress=progress) + Path(request.evidence_dir)/'previous-release', block_name=request.block_name, + progress=progress, stop=stop) + check_active() library_lock = LibraryLock(request.lmc_file, checkpoint=False, safe_to_release=clients_released) library_lock.__enter__() + check_active() except LibraryBusy as error: return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE') except (OSError, ValueError, RuntimeError, pywintypes.error) as error: @@ -691,14 +733,12 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, if library_lock is not None: library_lock.__exit__(None, None, None) finally: - if acquired: - win32event.ReleaseMutex(mutex) - mutex.Close() uninitialize_com(initialized) def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None): + from .operation_queue import remember_result timing = RunTimings() result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader, process_runner=process_runner, port_check=port_check, @@ -722,6 +762,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_ result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。') except (OSError, ValueError, KeyError): pass + remember_result(result) if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path: job_path = result.evidence_path.with_name('request.json') if job_path.is_file(): @@ -764,6 +805,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_ if result.status not in {"PASS", "SKIPPED"} else ""), error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution') if result.status not in {"PASS", "SKIPPED"} else "")) + remember_result(result) if result.evidence_path: _write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)}) _write_optional_evidence(result.evidence_path.with_name('performance.json'), { @@ -819,7 +861,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run except Exception: # Cleanup must never hide or change the verification result. pass - return replace(result, evidence_path=result_path) + from .operation_queue import remember_result + return remember_result(replace(result, evidence_path=result_path)) if stop is not None and stop.is_set(): return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。')) @@ -843,7 +886,9 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run 'detail': ' '.join(command)}) timing.begin('content_check') try: - exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent) + from .operation_queue import native_channel + with native_channel(): + exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent) finally: timing.end('content_check') combined = '\n'.join(part for part in (stdout, stderr) if part) @@ -915,17 +960,15 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run from .desktop_worker import DesktopWorker from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend from .dialog_models import DialogPhase - import win32event, win32con - mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') - lock_status = win32event.WaitForSingleObject(mutex, 0) - if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED): - mutex.Close() - return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。', - safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')) + from .operation_queue import check_active, native_channel + check_active() worker_launch_attempted = False + native = native_channel() + phase_entered = False try: timing.begin('worker_setup') with (root/'worker.log').open('w', encoding='utf-8') as log: + check_active() worker_launch_attempted = True if request.interactive_input: process = DesktopWorker( @@ -965,6 +1008,13 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run ) if stop is not None and stop.is_set(): raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB') + check_active() + try: + native.__enter__() + phase_entered = True + except RuntimeError as error: + return finish(VerifyResult('BLOCKED', stderr=str(error), safe_to_continue=False, + error_category='NATIVE_CHANNEL_BUSY')) Path(job['start_gate']).touch() while process.poll() is None: if stop is not None and stop.is_set(): @@ -1188,6 +1238,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run {'stage': 'session_observation', 'block': request.block_name, **state})) _notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name, 'detail': '已转后台观察,可查看状态'}) + if getattr(process, 'native_write_pending', lambda: None)() is not False: + from .operation_queue import hold_native_handoff + hold_native_handoff(process, process.session_record_path) + worker_exited = process.poll() is not None except Exception as error: close_errors.append(str(error)) if 'process' in locals(): @@ -1279,8 +1333,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run except OSError: pass finally: - win32event.ReleaseMutex(mutex) - mutex.Close() + if phase_entered: + native.__exit__(None, None, None) if result.status == 'PASS' and not request.diagnostic_only: timing.begin('library_flag') @@ -1290,9 +1344,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run if progress: _notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record}) try: - probe = read_library_flag_with_retry( - flag_reader or read_library_flag_if_available, Path(request.lmc_file), - request.block_name, on_attempt=record_attempt) + with native_channel(): + probe = read_library_flag_with_retry( + flag_reader or read_library_flag_if_available, Path(request.lmc_file), + request.block_name, on_attempt=record_attempt) result = apply_post_verify_status(result, probe) result = replace(result, bridge_attempts=tuple(bridge_records)) _write_optional_evidence(root/'bridge-status.json', bridge_records) @@ -1304,20 +1359,27 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run return finish(result) -def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None): +def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None, + context=None): """Run serially and stop when database/session safety is unconfirmed.""" if max_workers != 1: raise ValueError('Only serial Verify is supported for the shared Library Manager session') + if context is not None: + stop = stop if stop is not None else context.stop + progress = context.guard_callback(progress) if progress is not None else None if verifier is None: - verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available, stop=stop) # noqa: E731 + verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available, + stop=stop, context=context) # noqa: E731 results = [] queue_stop = False retained_libraries = set() for request in requests: - library_key = str(getattr(request, 'lmc_file', '__injected_verifier_scope__')).casefold() - if queue_stop or (stop is not None and stop.is_set()): + from .operation_queue import request_library_keys + library_keys = request_library_keys(request, context) + if (queue_stop or stop is not None and stop.is_set() + or context is not None and context.stop.is_set()): result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB') - elif library_key in retained_libraries: + elif library_keys.intersection(retained_libraries): result = VerifyResult('SKIPPED', stderr='同库原生会话尚未释放,未启动新的写入') else: try: @@ -1325,18 +1387,19 @@ def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=Non result = verifier(request) if not result.safe_to_continue: from .operation_lifecycle import recheck_failed_release - result = recheck_failed_release(request.lmc_file, request.block_name, result) + result = recheck_failed_release(request.lmc_file, request.block_name, result, stop=stop) return result result = item_executor(request, execute) if item_executor is not None else execute() except Exception as error: - from .operation_queue import QueueCancelled + from .operation_queue import QueueCancelled, TaskHandoff # An unexpected exception cannot establish database safety. - result = VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL', - stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled)) + result = (_handoff_verify_result(error) if isinstance(error, TaskHandoff) else + VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL', + stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled))) results.append(result) # Never start another writer after an unconfirmed native shutdown. if not result.safe_to_continue: - retained_libraries.add(library_key) + retained_libraries.update(library_keys) queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'} if queue_stop and stop is not None: stop.set() diff --git a/tests/test_batch_item_yield.py b/tests/test_batch_item_yield.py index 1ee8318..02d737f 100644 --- a/tests/test_batch_item_yield.py +++ b/tests/test_batch_item_yield.py @@ -71,7 +71,7 @@ def test_verify_failure_recheck_finishes_before_yielding_native_queue(qtbot,tmp_ try:return action() finally:active.pop() return run - def recheck(library,block,result): + def recheck(library,block,result,**kwargs): seen.append(bool(active)) return replace(result,safe_to_continue=True) monkeypatch.setattr(window,'_queued_action',queued) @@ -92,7 +92,7 @@ def test_management_failure_recheck_finishes_before_yielding_native_queue(tmp_pa active.append(True) try:return action() finally:active.pop() - def recheck(library,block,result): + def recheck(library,block,result,**kwargs): seen.append(bool(active)) return replace(result,safe_to_continue=True) monkeypatch.setattr(operation_lifecycle,'recheck_failed_release',recheck) diff --git a/tests/test_issue15_task_lifecycle.py b/tests/test_issue15_task_lifecycle.py new file mode 100644 index 0000000..d999ba5 --- /dev/null +++ b/tests/test_issue15_task_lifecycle.py @@ -0,0 +1,724 @@ +"""Cancellation must release scheduling, never invent native release.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Event, Thread +import time + +import pytest +import win32con +import win32event + +from cbb_refresh_verify import library_lock as locks +from test_management import setup as management_setup + + +@pytest.mark.parametrize('stage', ['read', 'archive']) +def test_stalled_checkpoint_does_not_own_native_channel_or_commit_after_stop(tmp_path, monkeypatch, stage): + library = tmp_path/'A/lib.lmc' + library.parent.mkdir() + library.touch() + with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease: + pass + marker = locks.blocker_path(library) + original = marker.read_bytes() + entered, resume, stop = Event(), Event(), Event() + read, write = Path.read_bytes, Path.write_bytes + def read_bytes(path): + if stage == 'read' and path == marker and not resume.is_set(): + entered.set() + assert resume.wait(5) + return read(path) + def write_bytes(path, data): + if stage == 'archive' and path.name.startswith('blocker-before-'): + entered.set() + assert resume.wait(5) + return write(path, data) + monkeypatch.setattr(Path, 'read_bytes', read_bytes) + monkeypatch.setattr(Path, 'write_bytes', write_bytes) + monkeypatch.setattr('cbb_refresh_verify.owner_probe.query_owners', lambda *_: []) + with ThreadPoolExecutor(1) as pool: + task = pool.submit(locks.release_retained_library, library, tmp_path/'evidence', stop=stop) + mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') + acquired = False + try: + assert entered.wait(3) + acquired = win32event.WaitForSingleObject(mutex, 0) in ( + win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED) + assert acquired, 'checkpoint I/O retained the host native channel' + if stage == 'archive': + import json + receipt = next((tmp_path/'evidence').rglob('release.json')) + assert json.loads(receipt.read_text())['lease_state'] != 'RELEASED' + stop.set() + resume.set() + with pytest.raises(InterruptedError): + task.result(5) + assert read(marker) == original + assert locks.has_retained_library(library) + finally: + stop.set() + resume.set() + if acquired: + win32event.ReleaseMutex(mutex) + mutex.Close() + try: + task.result(5) + except InterruptedError: + pass + with locks._guard: + owner = locks._owners.pop(lease.key, None) + if owner: + owner['handle'].Close() + + +def test_task_context_is_frozen_pure_and_has_an_independent_monotonic_stop(tmp_path, monkeypatch): + from dataclasses import FrozenInstanceError + from cbb_refresh_verify.operation_queue import BatchTaskContext + monkeypatch.setattr(Path, 'resolve', lambda *a, **k: pytest.fail('context performed filesystem admission')) + first = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',), targets=('A',)) + second = BatchTaskContext(library=str(tmp_path/'B.lmc')) + with pytest.raises(FrozenInstanceError): + first.library = second.library + first.cancel() + assert first.stop.is_set() and not second.stop.is_set() + assert not hasattr(first.stop, 'clear') + received = [] + first.guard_callback(received.append)('CANCELLED') + assert received == ['CANCELLED'] + first.close(timeout_seconds=0) + first.guard_callback(received.append)('late') + assert received == ['CANCELLED'] + + +def test_bounded_queue_handoff_preserves_scope_and_fences_late_native_launch(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff, native_phase + queue = NativeQueue() + first = BatchTaskContext(library=str(tmp_path/'A.lmc'), targets=('A',)) + second = BatchTaskContext(library=str(tmp_path/'B.lmc'), targets=('B',)) + entered, resume, finished = Event(), Event(), Event() + effects = [] + def stalled(): + try: + entered.set() + assert resume.wait(3) + with native_phase(): + effects.append('late native write') + finally: + finished.set() + try: + with pytest.raises(TaskHandoff) as error: + queue.run(stalled, context=first, timeout_seconds=.05) + assert entered.is_set() and error.value.library == first.library + assert first.snapshot()['pending'] == 1 + assert queue.run(lambda: 'B ran', context=second) == 'B ran' + resume.set() + assert finished.wait(2) + assert effects == [] + finally: + resume.set() + assert finished.wait(2) + + +def test_native_phase_is_never_yielded_at_metadata_deadline(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, native_phase + queue = NativeQueue() + context = BatchTaskContext(library=str(tmp_path/'A.lmc')) + entered, resume, finished = Event(), Event(), Event() + results = [] + def transaction(): + with native_phase(): + entered.set() + assert resume.wait(3) + return 'Remove and Add completed' + def run(): + results.append(queue.run(transaction, context=context, timeout_seconds=.02)) + finished.set() + thread = Thread(target=run) + thread.start() + try: + assert entered.wait(2) + assert not finished.wait(.1) + state = context.close(timeout_seconds=.02) + assert state['pending_native'] and queue.active + resume.set() + assert finished.wait(2) + assert results == ['Remove and Add completed'] + finally: + resume.set() + thread.join(2) + + +def test_stalled_ready_predicate_does_not_hold_queue_condition(): + from cbb_refresh_verify.operation_queue import NativeQueue, QueueCancelled + queue = NativeQueue() + entered, resume, stop, completed = Event(), Event(), Event(), Event() + effects = [] + def slow_ready(): + entered.set() + assert resume.wait(3) + return True + def first(): + try: + queue.run(lambda: effects.append('A'), ready=slow_ready, stop=stop) + except QueueCancelled: + pass + thread = Thread(target=first) + thread.start() + second = Thread(target=lambda: (queue.run(lambda: effects.append('B')), completed.set())) + try: + assert entered.wait(2) + second.start() + assert completed.wait(.5), 'one ready predicate froze every other queue ticket' + stop.set() + resume.set() + thread.join(2) + assert effects == ['B'] + finally: + stop.set() + resume.set() + thread.join(3) + second.join(3) + + +def test_verify_admission_handoff_unblocks_b_and_never_launches_after_read_returns(tmp_path, monkeypatch): + from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue + from cbb_refresh_verify.verify_runner import VerifyRequest, run_verify + import win32process + library = tmp_path/'A/lib.lmc' + library.parent.mkdir() + library.touch() + with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease: + pass + marker = locks.blocker_path(library) + entered, resume, returned = Event(), Event(), Event() + read = Path.read_text + def withheld(path, *args, **kwargs): + if path == marker: + entered.set() + assert resume.wait(3) + returned.set() + return read(path, *args, **kwargs) + monkeypatch.setattr(Path, 'read_text', withheld) + monkeypatch.setattr(win32process, 'CreateProcess', lambda *a, **k: pytest.fail('late native launch')) + context = BatchTaskContext(library=str(library), targets=('A',)) + events = [] + try: + result = run_verify(VerifyRequest(library, 'A', tmp_path/'evidence'), context=context, + admission_timeout_seconds=.05, progress=events.append) + assert entered.is_set() and result.status == 'TIMEOUT' + assert not result.safe_to_continue + assert result.release_proof['resource_scope']['operation_ids'] == [context.operation_id] + def unrelated(): + mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') + acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0 + try: + assert acquired + return 'B completed' + finally: + if acquired: win32event.ReleaseMutex(mutex) + mutex.Close() + assert NativeQueue().run(unrelated) == 'B completed' + count = len(events) + resume.set() + assert returned.wait(2) + limit = time.monotonic()+2 + while context.snapshot()['pending'] and time.monotonic() < limit: + time.sleep(.01) + assert context.snapshot()['pending'] == 0 + assert len(events) == count + finally: + resume.set() + context.close(timeout_seconds=2) + with locks._guard: + owner = locks._owners.pop(lease.key, None) + if owner: owner['handle'].Close() + + +def test_native_time_does_not_consume_next_metadata_phase_budget(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext, native_phase + context = BatchTaskContext(library=str(tmp_path/'A.lmc')) + stages = [] + def action(): + with native_phase(): + time.sleep(.12) + stages.append('saved') + time.sleep(.02) + with native_phase(): + stages.append('closed') + return stages + assert context.run(action, timeout_seconds=.06) == ['saved', 'closed'] + + +def test_native_channel_enforces_cross_thread_serialization_and_reentrancy(): + from cbb_refresh_verify.operation_queue import native_channel + with native_channel(): + with native_channel(): + with ThreadPoolExecutor(1) as pool: + def contend(): + with native_channel(): + pytest.fail('concurrent native writer admitted') + with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'): + pool.submit(contend).result(2) + with native_channel(): + pass + + +def test_maintenance_checkpoint_admission_does_not_hold_native_channel(tmp_path, monkeypatch): + from contextlib import nullcontext + library = tmp_path/'A/lib.lmc' + library.parent.mkdir() + library.touch() + with locks.LibraryLock(library, safe_to_release=lambda: False) as retained: + pass + marker = locks.blocker_path(library) + entered, resume = Event(), Event() + original = Path.read_bytes + def withheld(path): + if path == marker and not resume.is_set(): + entered.set() + assert resume.wait(3) + return original(path) + monkeypatch.setattr(Path, 'read_bytes', withheld) + def admit(): + with locks.LibraryLock.for_maintenance(library, tmp_path/'archive', lambda: nullcontext({})): + pass + with ThreadPoolExecutor(1) as pool: + task = pool.submit(admit) + mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795') + acquired = False + try: + assert entered.wait(2) + acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0 + assert acquired + finally: + if acquired: win32event.ReleaseMutex(mutex) + mutex.Close() + resume.set() + task.result(5) + with locks._guard: + owner = locks._owners.pop(retained.key, None) + if owner: owner['handle'].Close() + + +def test_native_handoff_keeps_real_mutex_until_bound_process_returns(tmp_path): + import subprocess + import sys + from cbb_refresh_verify.operation_queue import ( + BatchTaskContext, NativeQueue, TaskHandoff, hold_native_handoff, native_channel) + from cbb_refresh_verify.session_record import write_session_record + process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True) + context = BatchTaskContext(library=str(tmp_path/'A.lmc')) + record = tmp_path/'session.json' + write_session_record(record, dict(operation_id=context.operation_id, library=context.library, + worker_pid=process.pid)) + queue = NativeQueue() + effects = [] + def action(): + with native_channel(): + hold_native_handoff(process, record) + with native_channel(): + effects.append('late mutation') + try: + with pytest.raises(TaskHandoff): + queue.run(action, context=context) + assert process.poll() is None + assert context.snapshot()['pending_native'] + assert queue.run(lambda: 'pure read') == 'pure read' + with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'): + with native_channel(): + pytest.fail('writer overlapped handed-off native call') + process.communicate('\n', timeout=3) + limit = time.monotonic()+3 + while context.snapshot()['pending'] and time.monotonic() < limit: + time.sleep(.01) + assert context.snapshot()['pending'] == 0 + with native_channel(): + pass + assert effects == [] + finally: + if process.poll() is None: + process.communicate('\n', timeout=3) + context.close(timeout_seconds=3) + + +def test_worker_write_barrier_survives_supervisor_process_exit(tmp_path): + import json + import subprocess + import sys + from cbb_refresh_verify.operation_queue import native_channel + ready, finish, done = (tmp_path/name for name in ('ready', 'finish', 'done')) + child = '''import sys,time,os +from pathlib import Path +from cbb_refresh_verify.desktop_worker import worker_native_channel +with worker_native_channel(): + Path(sys.argv[1]).write_text(str(os.getpid())) + while not Path(sys.argv[2]).exists(): time.sleep(.02) +Path(sys.argv[3]).touch() +''' + parent = '''import subprocess,sys,time,json +from pathlib import Path +subprocess.Popen([sys.executable,'-c',sys.argv[1],*sys.argv[2:]]) +deadline=time.monotonic()+4 +while not Path(sys.argv[2]).exists() and time.monotonic()1 else 'Tip of the Day' + return 'OK' + monkeypatch.setattr(win32gui, 'EnumWindows', lambda fn, arg: [fn(h, arg) for h in sorted(visible)]) + monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda hwnd, fn, arg: + [fn(h, arg) for h in ((201, 202) if obstacle == 'duplicate' else (201,))] if hwnd == 200 else None) + monkeypatch.setattr(win32gui, 'GetClassName', classes.__getitem__) + monkeypatch.setattr(win32gui, 'GetWindowText', title) + monkeypatch.setattr(win32gui, 'GetDlgCtrlID', lambda hwnd: 1) + def post(hwnd, message, wparam, lparam): + messages.append((hwnd, message)) + if hwnd == 201: visible.remove(200) + elif hwnd == 100: records.clear() + monkeypatch.setattr(win32gui, 'PostMessage', post) + result = close_failed_startup(job, 12, timeout_seconds=0) + if obstacle: + assert result['status'] == 'RETAINED' and not messages + else: + assert result['status'] == 'CLOSED' + assert messages == [(201, win32con.BM_CLICK), (100, win32con.WM_CLOSE)] + + +def test_late_release_recheck_is_bounded_and_keeps_business_failure(tmp_path, monkeypatch): + from cbb_refresh_verify.operation_lifecycle import recheck_failed_release + from cbb_refresh_verify.verify_runner import VerifyResult + from cbb_refresh_verify.session_record import write_session_record + library = tmp_path/'A/lib.lmc' + library.parent.mkdir() + library.touch() + evidence = tmp_path/'evidence' + evidence.mkdir() + result = VerifyResult('FAIL', stderr='native business failure', safe_to_continue=False, + evidence_path=evidence/'result.json') + result.evidence_path.write_text('{}') + record = evidence/'session.json' + write_session_record(record, dict(operation_id='failed', library=str(library), block='A', processes=[])) + entered, resume = Event(), Event() + original = Path.open + def withheld(path, *args, **kwargs): + if path == record: + entered.set() + assert resume.wait(3) + return original(path, *args, **kwargs) + monkeypatch.setattr(Path, 'open', withheld) + try: + started = time.monotonic() + returned = recheck_failed_release(library, 'A', result, timeout_seconds=.05) + assert time.monotonic()-started < .8 + assert entered.is_set() and returned is result + assert returned.status == 'FAIL' and not returned.safe_to_continue + finally: + resume.set() + + +def test_batch_context_cancel_records_all_remaining_items_and_new_batch_can_run(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext + from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch + requests = [VerifyRequest(tmp_path/'lib.lmc', name, tmp_path/'evidence') for name in ('A', 'B', 'C')] + first = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A', 'B', 'C')) + events = [] + def verify(request): + first.cancel() + return VerifyResult('CANCELLED') + results = run_batch(requests, verifier=verify, context=first, + progress=lambda name, result: events.append((name, result.status))) + assert [r.status for r in results] == ['CANCELLED', 'SKIPPED', 'SKIPPED'] + assert events == [('A', 'CANCELLED'), ('B', 'SKIPPED'), ('C', 'SKIPPED')] + second = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A',)) + assert run_batch(requests[:1], verifier=lambda _: VerifyResult('PASS'), context=second)[0].status == 'PASS' + + +def test_batch_retention_matches_normalized_library_alias_not_business_failure(tmp_path): + from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch + library = tmp_path/'A/lib.lmc' + alias = tmp_path/'A/sub/../lib.lmc' + requests = [VerifyRequest(path, block, tmp_path/'evidence') for path, block in ( + (library, 'A'), (alias, 'B'), (tmp_path/'B/lib.lmc', 'C'), (tmp_path/'B/lib.lmc', 'D'))] + calls = [] + def verify(request): + calls.append(request.block_name) + return VerifyResult('FAIL', safe_to_continue=request.block_name != 'A') + results = run_batch(requests, verifier=verify) + assert calls == ['A', 'C', 'D'] + assert [r.status for r in results] == ['FAIL', 'SKIPPED', 'FAIL', 'FAIL'] + + +def test_queue_close_has_one_deadline_and_reports_pending_scope(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff + queue = NativeQueue() + context = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',)) + entered, resume, finished = Event(), Event(), Event() + def action(): + entered.set() + resume.wait(3) + def run(): + try: + queue.run(action, context=context) + except TaskHandoff: + pass + finally: + finished.set() + thread = Thread(target=run) + thread.start() + try: + assert entered.wait(2) + started = time.monotonic() + states = queue.close(timeout_seconds=.05) + assert time.monotonic()-started < .5 + assert states[0]['resource_scope']['aliases'] == ['Z:/A.lmc'] + assert states[0]['pending'] == 1 + assert finished.wait(1) + finally: + resume.set() + thread.join(2) + + +def test_real_management_transaction_does_not_handoff_between_remove_and_add(management_setup): + from dataclasses import replace + from test_management import execute + from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue + request, adapter, source = management_setup + original = adapter.remove + def slow_remove(*args): + result = original(*args) + time.sleep(.4) + return result + adapter.remove = slow_remove # Native external boundary; transaction stays real. + context = BatchTaskContext(library=str(request.library), targets=('Block',)) + result = NativeQueue().run(lambda: execute(replace(request, operation='update', + source=source, auto_verify=False), adapter), context=context, timeout_seconds=.2) + assert result.status == 'UPLOADED' + assert adapter.calls[0] == 'remove' and adapter.calls[1][0] == 'add' + + +def test_busy_native_channel_before_dispatch_does_not_retain_management_library(management_setup): + from test_management import execute + from cbb_refresh_verify.operation_queue import native_channel + request, adapter, _ = management_setup + entered, resume = Event(), Event() + def owner(): + with native_channel(): + entered.set() + resume.wait(3) + thread = Thread(target=owner) + thread.start() + try: + assert entered.wait(2) + result = execute(request, adapter) + assert result.safe_to_continue and not adapter.calls + with locks.LibraryLock(request.library): + pass + finally: + resume.set() + thread.join(3) + + +def test_only_explicit_repair_inspection_can_enter_while_worker_owns_writer_barrier(tmp_path): + import json + import os + from cbb_refresh_verify.desktop_worker import worker_native_channel, run_native_worker + from cbb_refresh_verify.operation_queue import native_channel + from cbb_refresh_verify.native_repair_ui import repair_worker_main + entered, resume = Event(), Event() + def owner(): + with worker_native_channel(): + entered.set() + resume.wait(3) + thread = Thread(target=owner) + thread.start() + job = dict(root=str(tmp_path), project=str(tmp_path/'A.prj'), database=str(tmp_path/'database'), + cancel_signal=str(tmp_path/'cancel'), supervisor_pid=os.getpid(), desktop='not-private') + path = tmp_path/'repair-request.json' + try: + assert entered.wait(2) + with native_channel(inspection_only=True): + pass + with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'): + with native_channel(): + pass + for mode in (False, 'true'): + path.write_text(json.dumps({**job, 'inspection_only': mode})) + with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'): + run_native_worker(repair_worker_main, path) + path.write_text(json.dumps({**job, 'inspection_only': True})) + with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'): + run_native_worker(lambda _: pytest.fail('arbitrary worker exempted'), path) + assert run_native_worker(repair_worker_main, path) == 1 + result = json.loads((tmp_path/'repair-result.json').read_text()) + assert result['native_started'] is False + assert 'REPAIR_PRIVATE_DESKTOP_REQUIRED' in result['detail'] + finally: + resume.set() + thread.join(3) + + +def test_inspection_session_does_not_take_over_retained_writer_lease(tmp_path): + import os + import win32api + from cbb_refresh_verify.desktop_worker import DesktopWorker + library = tmp_path/'A/lib.lmc' + library.parent.mkdir() + library.touch() + with locks.LibraryLock(library, safe_to_release=lambda: False) as lease: + pass + worker = object.__new__(DesktopWorker) + worker.handle, worker.pid = win32api.GetCurrentProcess(), os.getpid() + try: + worker.bind_session(dict(operation_id='inspection', operation='inspect', inspection_only=True, + library=str(library), project=str(tmp_path/'A.prj'), session_record_path=str(tmp_path/'session.json'))) + assert locks.has_retained_library(library) + from cbb_refresh_verify.session_record import read_session_record + record = read_session_record(tmp_path/'session.json', evidence_root=tmp_path) + assert record['worker_lease_handle'] is None + finally: + with locks._guard: + owner = locks._owners.pop(lease.key, None) + if owner: owner['handle'].Close() + + +def test_handoff_preserves_known_business_result_and_save_facts(tmp_path): + from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, remember_result + from cbb_refresh_verify.verify_runner import VerifyResult + context = BatchTaskContext(library=str(tmp_path/'A.lmc')) + native_result = VerifyResult('FAIL', stderr='native failure after Save', safe_to_continue=False, + write_outcome=dict(save_state='COMMITTED', close_state='UNKNOWN')) + resume, done = Event(), Event() + def finalize(): + remember_result(native_result) + try: + resume.wait(3) + finally: + done.set() + try: + with pytest.raises(TaskHandoff) as error: + context.run(finalize, timeout_seconds=.05) + assert error.value.result is native_result + assert error.value.result.write_outcome['save_state'] == 'COMMITTED' + assert error.value.result.status == 'FAIL' + from cbb_refresh_verify.verify_runner import _handoff_verify_result + returned = _handoff_verify_result(error.value) + assert returned.status == 'FAIL' and returned.stderr == native_result.stderr + assert returned.write_outcome == native_result.write_outcome and not returned.safe_to_continue + error.value.result = VerifyResult('PASS', write_outcome=native_result.write_outcome) + unfinished = _handoff_verify_result(error.value) + assert unfinished.status == 'FAIL' and unfinished.library_flag == 'UNKNOWN' + assert not unfinished.safe_to_continue + finally: + resume.set() + assert done.wait(2) + + +def test_registered_writer_owns_process_handle_after_callers_handle_closes(monkeypatch): + import subprocess + import sys + from cbb_refresh_verify import desktop_worker + monkeypatch.setattr(desktop_worker, '_automation_clients', []) + process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True) + entered, done = Event(), Event() + errors = [] + def run(): + try: + with desktop_worker.worker_native_channel(): + desktop_worker.register_native_writer(process) + process._handle.Close() + entered.set() + except Exception as error: + errors.append(error) + finally: + done.set() + thread = Thread(target=run) + thread.start() + try: + assert entered.wait(2) + assert not done.wait(.1), 'a closed caller handle released an unreturned writer' + with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'): + desktop_worker.assert_native_writer_idle() + finally: + process.stdin.write('\n') + process.stdin.flush() + process.stdin.close() + thread.join(3) + # The duplicate's observed exit is authoritative; the original Popen + # handle was deliberately invalidated, so do not poll it in __del__. + process.returncode = 0 + assert done.is_set() and not errors diff --git a/tests/test_late_release_continuation.py b/tests/test_late_release_continuation.py index 7ca8516..eea8fdd 100644 --- a/tests/test_late_release_continuation.py +++ b/tests/test_late_release_continuation.py @@ -17,7 +17,9 @@ def test_batch_rechecks_owned_failure_before_skipping_next_item(tmp_path,monkeyp monkeypatch.setattr(session_record,'inspect_session_record',lambda *a,**k:dict(status='UNCONFIRMED')) def proof(*a,**k): if not released:raise library_lock.LibraryBusy('shared database still occupied') - return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False) + return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False, + lease_state='RELEASED',blocked_resources=[], + resource_scope=dict(scope_state='OBSERVED',library=str(library))) monkeypatch.setattr(library_lock,'release_retained_library',proof) def verify(req): calls.append(req.block_name) diff --git a/tests/test_management.py b/tests/test_management.py index b7cddd5..c0a893a 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -215,7 +215,7 @@ def test_cancel_and_restart_never_replay(setup): from cbb_refresh_verify.management import run_management_batch, unfinished_operations request, adapter, _ = setup stop = Event(); stop.set() - assert run_management_batch([request], stop=stop, adapter=adapter) == [] + assert [r.status for r in run_management_batch([request], stop=stop, adapter=adapter)] == ['SKIPPED'] checkpoint = request.evidence_dir/'interrupted/checkpoint.json' checkpoint.parent.mkdir(parents=True) checkpoint.write_text('{"stage":"native_remove", "status":"RUNNING"}', encoding='utf-8') diff --git a/tests/test_native_management.py b/tests/test_native_management.py index 6a60632..9c89834 100644 --- a/tests/test_native_management.py +++ b/tests/test_native_management.py @@ -313,8 +313,12 @@ def test_private_worker_timeout_retains_native_and_never_kills(tmp_path, monkeyp def test_hidden_worker_and_probe_dispatch_before_qt(monkeypatch, tmp_path): import cbb_refresh_verify.native_management as native from cbb_refresh_verify.application import main + from cbb_refresh_verify import desktop_worker + # This dispatch represents a new worker process, not other tests' mocked clients. + monkeypatch.setattr(desktop_worker, '_automation_clients', []) seen = [] monkeypatch.setattr(native, 'management_worker_main', lambda p: (seen.append(p) or 17)) + (tmp_path/'job.json').write_text('{}') assert main(['--management-worker', str(tmp_path/'job.json')]) == 17 assert seen == [tmp_path/'job.json'] diff --git a/tests/test_review414_write_admission.py b/tests/test_review414_write_admission.py index 795713e..a6a3947 100644 --- a/tests/test_review414_write_admission.py +++ b/tests/test_review414_write_admission.py @@ -94,7 +94,7 @@ def test_user_can_close_unknown_client_before_confirming(tmp_path,monkeypatch): _idle_check(*paths,confirmation=WriteAdmission(prompt=prompt),evidence=tmp_path/'evidence',block='Block') -@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED'])]) +@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED','SKIPPED'])]) def test_batch_continues_only_after_skip_or_confirmed_release(tmp_path,monkeypatch,first,expected): from dataclasses import replace from test_management import FileAdapter,project @@ -149,7 +149,7 @@ def test_cancel_batch_also_stops_default_api_without_a_caller_event(tmp_path,mon request=ManagementRequest('add',library,'One',source=source,auto_verify=False,evidence_dir=tmp_path/'evidence') adapter=FileAdapter() results=run_management_batch([request,replace(request,name='Two')],adapter=adapter,version_prompt=prompt) - assert [r.status for r in results]==['CANCELLED'] + assert [r.status for r in results]==['CANCELLED','SKIPPED'] assert len(choices)==1 and not adapter.calls