fix: bound task handoff and preserve native writer ownership

This commit is contained in:
阿白叔 committed 2026-11-16 18:57:36 +08:00
1 parent 0440b2efbf
commit ddff6bd857
18 files changed
+1397 -118

No files matched your search

+8 -4
View File
@@ -98,13 +98,16 @@ def main(argv: Sequence[str] | None = None) -> int:
return owner_probe_main(args.owner_probe)
if args.native_backup_worker:
from .native_backup import native_backup_worker_main
return native_backup_worker_main(args.native_backup_worker)
from .desktop_worker import run_native_worker
return run_native_worker(native_backup_worker_main, args.native_backup_worker)
if args.management_worker:
from .native_management import management_worker_main
return management_worker_main(args.management_worker)
from .desktop_worker import run_native_worker
return run_native_worker(management_worker_main, args.management_worker)
if args.repair_worker:
from .native_repair_ui import repair_worker_main
return repair_worker_main(args.repair_worker)
from .desktop_worker import run_native_worker
return run_native_worker(repair_worker_main, args.repair_worker)
if args.collect_failure_logs:
from .verify_diagnostics import failure_logs_main
return failure_logs_main(args.collect_failure_logs)
@@ -116,7 +119,8 @@ def main(argv: Sequence[str] | None = None) -> int:
import contextlib
with args.verify_worker.with_name('worker.log').open('a', encoding='utf-8') as log:
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log):
return worker_main(args.verify_worker)
from .desktop_worker import run_native_worker
return run_native_worker(worker_main, args.verify_worker)
context = configure(resolve_data_root(args.data_root)); logger = context.logger; logger.info("application_start")
try:
if args.batch_lmc:
+114 -1
View File
@@ -16,6 +16,110 @@ _automation_clients = []
_automation_clients_lock = threading.Lock()
_supervisor_lost = threading.Event()
_native_phases = {}
WRITER_MUTEX = r'Local\CbbRefreshVerify795NativeWriter'
def assert_native_writer_idle():
mutex = win32event.CreateMutex(None, False, WRITER_MUTEX)
acquired = False
try:
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回,未启动新的写入。')
finally:
if acquired: win32event.ReleaseMutex(mutex)
mutex.Close()
class _RegisteredWriter:
def __init__(self, handle):
import win32api
current = win32api.GetCurrentProcess()
self.handle = win32api.DuplicateHandle(current, handle, current, 0, False,
win32con.DUPLICATE_SAME_ACCESS)
self.returncode = None
def poll(self):
if self.returncode is None and win32event.WaitForSingleObject(self.handle, 0) == win32con.WAIT_OBJECT_0:
self.returncode = win32process.GetExitCodeProcess(self.handle)
self.handle.Close()
return self.returncode
def register_native_writer(process):
handle = getattr(process, 'handle', None) or getattr(process, '_handle', None)
registered = _RegisteredWriter(handle) if handle is not None else process
with _automation_clients_lock:
_automation_clients.append(registered)
return process
@contextmanager
def worker_native_channel(*, lease_handle=None, inspection_only=False):
"""The native worker, not its disposable supervisor, owns write exclusion."""
import win32api
mutex = win32event.CreateMutex(None, False, WRITER_MUTEX)
acquired = False
try:
if inspection_only is not True:
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回。')
yield
finally:
if acquired:
# Only registered Save/Close/Repair clients extend the boundary;
# an idle PCB shell or a business failure is not a pending write.
warned = False
while True:
with _automation_clients_lock:
clients = tuple(_automation_clients)
try:
released = all(client.poll() is not None for client in clients)
except Exception:
released = False
if not warned:
logging.getLogger(__name__).warning('Native writer exit is unconfirmed', exc_info=True)
warned = True
if released:
break
threading.Event().wait(.1)
win32event.ReleaseMutex(mutex)
mutex.Close()
if lease_handle is not None:
win32api.CloseHandle(int(lease_handle))
def run_native_worker(action, request_path):
"""Wait for durable parent admission before acquiring the worker barrier."""
import json
import time
job = json.loads(Path(request_path).read_text(encoding='utf-8'))
gate = Path(job['start_gate']) if job.get('start_gate') else None
if gate is not None:
deadline = time.monotonic()+30
while not gate.is_file():
if (job.get('cancel_signal') and Path(job['cancel_signal']).exists()
or time.monotonic() >= deadline):
raise RuntimeError('WORKER_CANCELLED: 未取得启动许可。')
threading.Event().wait(.05)
lease_handle = None
if job.get('session_record_path'):
from .session_record import read_session_record
path = Path(job['session_record_path'])
record = read_session_record(path, evidence_root=path.parent)
if (record['operation_id'] != job['operation_id']
or not any(p['pid'] == os.getpid() for p in record.get('processes', ()) )):
raise RuntimeError('WORKER_ADMISSION_IDENTITY_CHANGED')
lease_handle = record.get('worker_lease_handle')
inspection = (action.__module__ == 'cbb_refresh_verify.native_repair_ui'
and action.__name__ == 'repair_worker_main' and job.get('inspection_only') is True)
with worker_native_channel(lease_handle=lease_handle, inspection_only=inspection):
if job.get('cancel_signal') and Path(job['cancel_signal']).exists():
raise RuntimeError('WORKER_CANCELLED: 已取消原生启动。')
return action(request_path)
def _phase_pending(phase):
@@ -56,6 +160,8 @@ def _record_write_pending(record_path, operation_id=None):
def launch_save_helper(command, *, start_gate=None, identity_path=None, operation_id=None, **kwargs):
"""Register the save client atomically with respect to supervisor loss."""
with _automation_clients_lock:
from .operation_queue import check_active
check_active()
if _supervisor_lost.is_set():
raise RuntimeError('VERIFY_CANCELLED: 监督器已退出,未启动新的保存客户端')
helper = subprocess.Popen(command, **kwargs)
@@ -160,6 +266,8 @@ class DesktopWorker:
startup = win32process.STARTUPINFO()
# subprocess.STARTUPINFO does NOT pass lpDesktop to CreateProcess.
startup.lpDesktop = startup_desktop_name(name, interactive_input=self.interactive_input)
from .operation_queue import check_active
check_active()
self.handle, thread, self.pid, _ = win32process.CreateProcess(
command[0], subprocess.list2cmdline(command), None, None, False,
win32con.CREATE_NO_WINDOW, None, None, startup)
@@ -199,6 +307,8 @@ class DesktopWorker:
raise
def start_companion(self, command):
from .operation_queue import check_active
check_active()
startup = win32process.STARTUPINFO()
startup.lpDesktop = startup_desktop_name(
self.name, interactive_input=getattr(self, 'interactive_input', False))
@@ -283,7 +393,10 @@ class DesktopWorker:
def bind_session(self, job):
from .session_record import register_worker_session
register_worker_session(self, job)
from .library_lock import duplicate_worker_lease
lease = (None if job.get('inspection_only') is True else
duplicate_worker_lease(job['library'], self.handle))
register_worker_session(self, {**job, 'worker_lease_handle': lease})
def native_write_pending(self):
return _record_write_pending(getattr(self, 'session_record_path', None))
+27
View File
@@ -53,6 +53,14 @@ def _cancel_target(snapshot):
return None
def _tip_target(snapshot):
from .dialog_agent import DialogActionExecutor
if snapshot.title != 'Tip of the Day' or snapshot.window_class != '#32770':
return None
target = DialogActionExecutor._target(snapshot, {'control_id': 1})
return target if target is not None and target.class_name == 'Button' else None
def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: None,
timeout_seconds=20, launcher_process=None):
"""Normally close our empty PCB after a diagnosed database startup failure.
@@ -84,6 +92,7 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event:
declined_hwnd = None
declined_kind = None
cancelled_open = False
dismissed_tips = set()
while True:
records = [r for r in _process_details('ExpeditionPCB.exe') if r[0] == pcb_pid]
if not records:
@@ -121,6 +130,24 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event:
if declined_hwnd is not None and declined_hwnd not in windows:
cancelled_open = True
result['cancelled_prompt'] = declined_kind
dismissed = False
for hwnd in windows:
if hwnd in dismissed_tips or win32gui.GetClassName(hwnd) != '#32770':
continue
snapshot = _startup_snapshot(hwnd, pcb_pid)
target = _tip_target(snapshot)
if target is None:
continue
current = _startup_snapshot(hwnd, pcb_pid)
if current != snapshot or _tip_target(current) != target:
raise ValueError('STARTUP_TIP_CHANGED')
win32gui.PostMessage(target.handle, win32con.BM_CLICK, 0, 0)
dismissed_tips.add(hwnd)
dismissed = True
emit(stage='startup_tip_dismissed', pcb_pid=pcb_pid, hwnd=hwnd)
break
if dismissed:
continue # Prove the Tip disappeared; it does not prove a cancelled open.
if declined_hwnd is None:
for hwnd in windows:
if win32gui.GetClassName(hwnd) != '#32770':
+32 -19
View File
@@ -20,6 +20,21 @@ def _library_guard(key):
return _library_guards.setdefault(key, threading.RLock())
def duplicate_worker_lease(library, process_handle):
"""Keep the exact admitted exclusive file object alive across parent exit."""
import win32api, win32con
lease = LibraryLock(library)
with _library_guard(lease.key):
owner = _owners.get(lease.key)
if owner is None:
return None
if owner['thread'] != threading.get_ident() or owner['retained'] or not owner['depth']:
raise LibraryBusy('不能将其他任务的库锁交给工作进程。')
remote = win32api.DuplicateHandle(win32api.GetCurrentProcess(), owner['handle'],
process_handle, 0, False, win32con.DUPLICATE_SAME_ACCESS)
return remote.Detach() # This handle belongs to the target process only.
class LibraryBusy(OSError):
pass
@@ -407,14 +422,12 @@ def _owner_is_blocking(item, library, roots):
def _enter_maintenance(lease):
"""Transfer only a valid known lease, after project admission under lock."""
import win32con, win32event, win32file
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
import win32con, win32file
from .operation_queue import check_active
handle = None
existing = None
try:
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired: raise LibraryBusy('已有 CBB 操作运行,维护不能接续')
check_active()
with _library_guard(lease.key):
marker = blocker_path(lease.library)
if marker.with_suffix('.pending').exists():
@@ -449,6 +462,7 @@ def _enter_maintenance(lease):
with saved.open('xb') as stream:
stream.write(old); stream.flush(); os.fsync(stream.fileno())
if saved.read_bytes() != old: raise OSError('原检查点归档校验失败')
check_active()
lease_id = uuid4().hex
_persist_blocker(lease.library, 'OPERATION_IN_PROGRESS', lease_id=lease_id)
_owners[lease.key] = {'thread': threading.get_ident(), 'depth': 1, 'handle': handle,
@@ -456,12 +470,9 @@ def _enter_maintenance(lease):
'release_probe': lease.safe_to_release, 'block_name': None}
lease.maintenance_proof = proof
lease.entered = True
lease.maintenance_mutex = mutex
return lease
except Exception:
if handle is not None and existing is None: handle.Close()
if acquired: win32event.ReleaseMutex(mutex)
mutex.Close()
raise
@@ -550,7 +561,13 @@ def _prove_library_released(library, block_name=None, *, block_names=(), allow_u
# and compare the exact address/project binding before commit.
stack.close()
from .icdb_ownership import inspect_release_addresses
inspections = inspect_release_addresses(files, stop=stop)
# Checkpoint I/O is per-library; only native inspection owns
# the host channel. The sidecar still excludes other writers.
from .operation_queue import native_channel
with native_channel(inspection_only=True):
if stop is not None and stop.is_set():
raise InterruptedError('已停止释放检查')
inspections = inspect_release_addresses(files, stop=stop)
try:
current = stack.enter_context(hold_release_files(library, roots, stop=stop))
except OSError as error:
@@ -582,7 +599,7 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
The upload sidecar is never deleted. This acknowledges process release,
not the correctness of a partially changed library or its recovery state.
"""
import win32con, win32event, win32file
import win32con, win32file
library = Path(library).resolve(strict=True)
from .session_record import (retained_session_snapshot, acknowledge_library_release,
reconciled_retained_sessions)
@@ -594,15 +611,10 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
if evidence.is_relative_to(library.parent) or library.parent.is_relative_to(evidence):
raise ValueError('释放检查证据必须位于目标库之外。')
lease = LibraryLock(library)
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
external = None
try:
if stop is not None and stop.is_set():
raise InterruptedError('已停止释放检查')
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
raise LibraryBusy('另一个 CBB 操作仍在运行。')
with _library_guard(lease.key):
owner = _owners.get(lease.key)
if owner and (not owner['retained'] or owner['depth']):
@@ -675,7 +687,8 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
'block_names': list(scoped_blocks),
'lease_id': owner['lease_id'] if owner else None,
'operation_ids': [item['operation_id'] for item in reconciled_sessions]}
(root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
pending = {**record, 'status': 'RELEASE_PENDING', 'lease_state': 'PENDING'}
(root/'release.json').write_text(json.dumps(pending, ensure_ascii=False, indent=2), encoding='utf-8')
for index, checkpoint in enumerate(checkpoints):
if checkpoint.exists():
data = checkpoint.read_bytes()
@@ -683,15 +696,15 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
archived.write_bytes(data)
if archived.read_bytes() != data:
raise OSError('阻止记录归档校验失败,保持库锁。')
if stop is not None and stop.is_set():
raise InterruptedError('已停止释放检查')
checkpoint.unlink() # Checkpoint JSON only; never .lock.
if owner:
owner['handle'].Close()
del _owners[lease.key]
(root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
acknowledge_library_release(record, matching_retained, block_name=scoped)
return record
finally:
if external is not None:
external.Close()
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
+34 -33
View File
@@ -173,7 +173,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
write_admission=None):
from .verify_runner import VerifyRequest, run_verify
from .backup_recovery import _idle_check
import win32con, win32event
from contextlib import ExitStack
from .operation_queue import check_active, native_channel, task_stop
stop = task_stop(stop)
import pythoncom
if adapter is None:
from .native_management import NativeManagementAdapter
@@ -190,8 +192,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
history = []
mutated = uploaded = removed = False
owner_safe = True
mutex = None
acquired = False
release_proof = None
original = None
change_report = None
@@ -199,12 +199,13 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
nonlocal owner_safe
if stop is not None and stop.is_set():
raise InterruptedError('MANAGEMENT_CANCELLED: 已停止后续原生操作')
owner_safe = False
try:
result = callback(*args)
except Exception as error:
owner_safe = getattr(error, 'safe_to_continue', False) is True
raise
with native_channel():
owner_safe = False
try:
result = callback(*args)
except Exception as error:
owner_safe = getattr(error, 'safe_to_continue', False) is True
raise
owner_safe = result.safe_to_continue
return result
def checkpoint(next_stage, **details):
@@ -292,14 +293,10 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
root.mkdir(parents=True, exist_ok=False)
report = root/'checkpoint.json'
checkpoint('release_check')
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
return finish('BLOCKED', '已有 CBB 校验/管理/恢复任务运行;本次未启动。')
from .library_lock import recheck_previous_release
release_proof = recheck_previous_release(library, root/'previous-release', block_name=request.name,
required=request.check_previous_release or bool(unfinished_target
and unfinished_target.get('safe_to_continue') is not True), progress=progress)
and unfinished_target.get('safe_to_continue') is not True), progress=progress, stop=stop)
if release_proof is not None:
checkpoint('release_check', release_proof=release_proof)
if unfinished_target:
@@ -308,8 +305,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
adapter.bind_library_mapping(request.mapped_drive, request.mapped_remote_root)
if hasattr(adapter,'bind_write_admission'):
adapter.bind_write_admission(write_admission if native_idle_check else None)
check_active()
with LibraryLock(library, block_name=request.name, safe_to_release=lambda: owner_safe or (
callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))):
callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))), ExitStack() as native_stack:
work = library.parent/'ReusableBlocksLP'/request.name/'Work'
target_project, target_pcb = work/(request.name+'.prj'), work/'PCB/pcb.pcb'
if request.operation != 'add':
@@ -324,7 +322,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
if source:
idle_check(library, source, pcb)
checkpoint('catalog_before')
owner_safe = False
before = invoke(adapter.catalog, library, root/'before')
owner_safe = before.safe_to_continue
if not owner_safe or Path(before.library).resolve() != canonical_library or before.flow != 1:
@@ -366,6 +363,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
if request.operation in ('remove', 'update'):
if stop is not None and stop.is_set():
return finish('CANCELLED', '已取消,未开始移除', owner_safe)
native_stack.enter_context(native_channel())
checkpoint('native_remove')
mutated = True; owner_safe = False
native = invoke(adapter.remove, request, root/'remove')
@@ -385,6 +383,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
if stop is not None and stop.is_set():
return finish('RECOVERY_REQUIRED' if mutated else 'CANCELLED',
'已停止后续导入;已发生的原生变更需要核对', owner_safe)
if not removed:
native_stack.enter_context(native_channel())
checkpoint('native_add')
mutated = True; owner_safe = False
native = invoke(adapter.add, staged, root/'add')
@@ -413,6 +413,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
return finish('RECOVERY_REQUIRED', '原生新增后名称、分区或工程关联未确认一致。\n'+native.detail, owner_safe)
uploaded = True
checkpoint('uploaded')
native_stack.close()
if not request.auto_verify:
return finish('UPLOADED', '上传已确认;本批次未启用自动校验。\n'+native.detail)
# Adapter catalog cannot return until its exact native owner exits.
@@ -442,29 +443,29 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
status = 'BLOCKED'
return finish(status, str(error), owner_safe)
finally:
if mutex is not None:
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
uninitialize_com(initialized)
def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, **kwargs):
def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, context=None, **kwargs):
from threading import Event
from .write_admission import WriteAdmission
from .operation_queue import QueueCancelled, TaskHandoff, request_library_keys
if context is not None:
stop = stop if stop is not None else context.stop
progress = context.guard_callback(progress) if progress else None
if stop is None:
stop = Event()
kwargs.setdefault('write_admission',WriteAdmission(prompt=kwargs.get('version_prompt'),stop=stop))
results = []
retained_libraries = set()
cancelled = False
for request in requests:
if stop is not None and stop.is_set():
break
key = str(request.library).casefold()
if key in retained_libraries:
keys = request_library_keys(request, context)
stopped = cancelled or stop.is_set() or context is not None and context.stop.is_set()
if stopped or keys.intersection(retained_libraries):
result = ManagementResult(request.operation, request.library, request.name, request.partition,
request.source, request.auto_verify, 'complete', 'SKIPPED', True,
'同库原生会话尚未释放,未启动新的写入')
'队列已停止,未操作此 CBB' if stopped else '同库原生会话尚未释放,未启动新的写入')
results.append(result)
if progress:
progress({'stage': result.status, 'block': result.name, 'detail': result.detail})
@@ -473,21 +474,21 @@ def run_management_batch(requests, *, stop=None, progress=None, item_executor=No
result = run_management(request, progress=progress, stop=stop, **kwargs)
if not result.safe_to_continue:
from .operation_lifecycle import recheck_failed_release
result = recheck_failed_release(request.library, request.name, result)
result = recheck_failed_release(request.library, request.name, result, stop=stop)
return result
from .operation_queue import QueueCancelled
try:
result = item_executor(request, action) if item_executor is not None else action()
except QueueCancelled as error:
results.append(ManagementResult(request.operation, request.library, request.name, request.partition,
request.source, request.auto_verify, 'complete', 'CANCELLED', True, str(error)))
break
result = ManagementResult(request.operation, request.library, request.name, request.partition,
request.source, request.auto_verify, 'complete',
'TIMEOUT' if isinstance(error, TaskHandoff) and not error.cancelled else 'CANCELLED',
not isinstance(error, TaskHandoff), str(error))
results.append(result)
if progress:
progress({'stage': result.status, 'block': result.name, 'detail': result.detail,
'evidence_path': str(result.evidence_path or '')})
if not result.safe_to_continue:
retained_libraries.add(key)
retained_libraries.update(keys)
if result.status == 'CANCELLED':
break
cancelled = True
return results
@@ -1285,6 +1285,8 @@ def management_worker_main(request_path):
except (FileNotFoundError, PermissionError):
result['safe_to_continue'] = True
raise
from .desktop_worker import register_native_writer
register_native_writer(process)
update_worker_state(job, 'NATIVE_CALL', identities=[('native', process.pid)], continuation=True)
try:
_write_json(root/'native-process.json', {'pid': process.pid, 'command': command})
+27 -3
View File
@@ -264,15 +264,34 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr
job_path = root/'repair-request.json'
cancel = root/'cancel-repair.signal'
desktop = 'CbbRepair_'+uuid4().hex[:12]
job_path.write_text(json.dumps({'project': str(target.project), 'database': str(target.database),
from .operation_queue import current_context
context = current_context()
library = (maintenance_context or {}).get('library') or (context.library if context else str(target.project))
job = {'project': str(target.project), 'database': str(target.database),
'root': str(root), 'mentor_bin': str(mentor_bin or ''), 'timeout': timeout_seconds,
'cancel_signal': str(cancel), 'supervisor_pid': os.getpid(), 'desktop': desktop,
'maintenance_context': maintenance_context, 'inspection_only': inspection_only},
ensure_ascii=False), encoding='utf-8')
'maintenance_context': maintenance_context, 'inspection_only': inspection_only,
'library': library, 'operation_id': root.name+'-'+uuid4().hex[:8],
'operation': 'inspect' if inspection_only else 'repair',
'block': (maintenance_context or {}).get('block'),
'session_record_path': str(root/'session.json'), 'start_gate': str(root/'start-repair.signal')}
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
command = ([sys.executable] if getattr(sys, 'frozen', False) else
[sys.executable, '-m', 'cbb_refresh_verify']) + ['--repair-worker', str(job_path)]
from .operation_queue import native_channel, task_stop
with native_channel(inspection_only=inspection_only is True):
return _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, task_stop(stop), job)
def _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, stop, job):
from .desktop_worker import DesktopWorker
worker = DesktopWorker(command, desktop)
worker.bind_cancellation(cancel)
if hasattr(worker, 'bind_session'):
worker.bind_session(job)
from .operation_queue import check_active
check_active()
Path(job['start_gate']).touch()
# Worker admission includes filesystem/ownership checks before the native
# timeout begins. Bound that preparation separately; never extend on polls.
deadline = time.monotonic()+60+max(1, float(timeout_seconds))+5
@@ -307,4 +326,9 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr
result['status'] = 'UNCONFIRMED'
return {**result, 'evidence_path': str(root)}
finally:
if worker.poll() is None and getattr(worker, 'session_record_path', None):
from .session_record import handoff_session
from .operation_queue import hold_native_handoff
handoff_session(worker.session_record_path, evidence_root=root, process=worker)
hold_native_handoff(worker, worker.session_record_path)
worker.close()
@@ -335,6 +335,8 @@ def repair_worker_main(job_path):
if Path(job['cancel_signal']).exists():
raise ValueError('REPAIR_CANCELLED_BEFORE_LAUNCH')
process = subprocess.Popen([str(executable), '-p', str(project)], cwd=str(project.parent))
from .desktop_worker import register_native_writer
register_native_writer(process)
result['native_started'] = True
result['native_pid'] = process.pid
emit(stage='native_started', pid=process.pid, project=str(project))
+19 -3
View File
@@ -2,10 +2,19 @@
from collections.abc import Sequence
def recheck_failed_release(library, block, result):
def recheck_failed_release(library, block, result, *, stop=None, timeout_seconds=10):
"""Refresh resource facts after a late exit; keep the original business failure."""
if result.safe_to_continue or not result.evidence_path:
return result
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
if current_context() is None:
context = BatchTaskContext(library=str(library), targets=(block,))
try:
return context.run(lambda: recheck_failed_release(library, block, result, stop=stop),
stop=stop, timeout_seconds=timeout_seconds)
except TaskHandoff:
return result
stop = task_stop(stop)
from pathlib import Path
from dataclasses import replace
import json
@@ -37,9 +46,16 @@ def recheck_failed_release(library, block, result):
bound.append(candidate)
if not bound:
return result # A generic timeout without a bound owner is not an exit fact.
proof = release_retained_library(library, evidence.parent/'late-release', block_name=block)
if proof.get('status') != 'RELEASE_CONFIRMED' or Path(proof.get('library', '')).resolve() != Path(library).resolve():
from .operation_queue import check_active
check_active()
proof = release_retained_library(library, evidence.parent/'late-release', block_name=block, stop=stop)
scope = proof.get('resource_scope')
if (proof.get('status') != 'RELEASE_CONFIRMED' or proof.get('lease_state') != 'RELEASED'
or not isinstance(scope, dict) or scope.get('scope_state') != 'OBSERVED'
or proof.get('blocked_resources') != []
or Path(proof.get('library', '')).resolve() != Path(library).resolve()):
return result
check_active()
evidence.with_name('late-release-result.json').write_text(json.dumps(dict(
business_status=result.status, safe_to_continue=True, release_proof=proof),
ensure_ascii=False, default=str, indent=2), 'utf-8')
+293 -9
View File
@@ -1,36 +1,320 @@
"""Cooperative FIFO admission for this application's native operations."""
from threading import Condition
from contextlib import contextmanager
from dataclasses import dataclass, field
import ntpath
from threading import Condition, Event, RLock, Thread, local
import time
from uuid import uuid4
class QueueCancelled(RuntimeError):
pass
class TaskHandoff(QueueCancelled):
"""The call is still observable; this is never a release receipt."""
def __init__(self, context, *, cancelled=False, result=None):
self.operation_id = context.operation_id
self.library = context.library
self.cancelled = cancelled
self.scope = context.scope
self.result = result
super().__init__('已交给后台观察;当前资源尚未确认释放。')
class _StopSignal:
def __init__(self):
self._event = Event()
def set(self):
self._event.set()
def is_set(self):
return self._event.is_set()
def wait(self, timeout=None):
return self._event.wait(timeout)
_local = local()
_tasks_guard = RLock()
_calls = {}
def _path_key(path):
# Identity was bound by the caller; queue predicates must never resolve a share.
return ntpath.normcase(ntpath.normpath(str(path)))
def request_library_keys(request, context=None):
library = str(getattr(request, 'lmc_file', getattr(request, 'library', '__injected_verifier_scope__')))
keys = {_path_key(library)}
drive, remote = getattr(request, 'mapped_drive', ''), getattr(request, 'mapped_remote_root', '')
if drive and remote and ntpath.splitdrive(library)[0].casefold() == drive.rstrip('\\/').casefold():
keys.add(_path_key(ntpath.join(remote, ntpath.splitdrive(library)[1].lstrip('\\/'))))
if context is not None:
aliases = {_path_key(v) for v in (context.library, *context.aliases)}
if keys.intersection(aliases):
keys.update(aliases)
return keys
@dataclass(eq=False)
class _Call:
context: object
external_stop: object = None
done: Event = field(default_factory=Event)
abandoned: bool = False
native_depth: int = 0
native_handoff: str = ''
phase_started: float = field(default_factory=time.monotonic)
result: object = None
last_result: object = None
error: BaseException | None = None
def is_set(self):
return (self.abandoned or self.context.stop.is_set()
or self.external_stop is not None and self.external_stop.is_set())
def current_context():
call = getattr(_local, 'call', None)
return call.context if call is not None else None
def task_stop(fallback=None):
return getattr(_local, 'call', None) or fallback
def check_active():
call = getattr(_local, 'call', None)
if call is not None and call.is_set():
raise QueueCancelled('任务已取消,未提交新的原生操作。')
def remember_result(result):
call = getattr(_local, 'call', None)
if call is not None:
with _tasks_guard:
call.last_result = result
return result
def hold_native_handoff(process, session_record_path):
"""Keep this thread's mutex while a recorded native client is unreturned.
The caller registers the observer first. Only UI/scheduling is handed off;
this thread and its native channel remain alive until that exact handle exits.
"""
call = getattr(_local, 'call', None)
if call is not None:
with _tasks_guard:
if not call.native_depth:
raise RuntimeError('Native handoff requires an owned native phase')
call.native_handoff = str(session_record_path)
while process.poll() is None:
Event().wait(.1)
@contextmanager
def native_phase():
"""Never abandon a transaction between native writes (including Remove/Add)."""
call = getattr(_local, 'call', None)
with _tasks_guard:
check_active()
if call is not None:
call.native_depth += 1
try:
yield
finally:
if call is not None:
with _tasks_guard:
call.native_depth -= 1
call.phase_started = time.monotonic()
@contextmanager
def native_channel(*, inspection_only=False):
"""Shared by Verify, Repair and backup native clients; reentrant per thread."""
import win32con, win32event
with native_phase():
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
try:
check_active()
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
raise RuntimeError('NATIVE_CHANNEL_BUSY: 另一个原生操作仍在运行。')
from .desktop_worker import assert_native_writer_idle
if inspection_only is not True:
assert_native_writer_idle()
yield
finally:
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
@dataclass(frozen=True)
class BatchTaskContext:
library: str
aliases: tuple[str, ...] = ()
targets: tuple[str, ...] = ()
resources: tuple[str, ...] = ()
operation_id: str = field(default_factory=lambda: uuid4().hex)
stop: _StopSignal = field(default_factory=_StopSignal, repr=False, compare=False)
_closed: Event = field(default_factory=Event, repr=False, compare=False)
def __post_init__(self):
object.__setattr__(self, 'library', str(self.library))
for name in ('aliases', 'targets', 'resources'):
object.__setattr__(self, name, tuple(str(v) for v in getattr(self, name)))
@property
def scope(self):
return dict(scope_state='DECLARED', library=self.library, aliases=list(self.aliases),
block_names=list(self.targets), resources=list(self.resources),
operation_ids=[self.operation_id])
@property
def state(self):
return self.snapshot()['state']
def snapshot(self):
with _tasks_guard:
calls = [call for call in _calls.values() if call.context is self]
pending = sum(not call.done.is_set() for call in calls)
return dict(operation_id=self.operation_id, resource_scope=self.scope,
state='CLOSED' if self._closed.is_set() else
'CANCELLED' if self.stop.is_set() else
'HANDED_OFF' if any(c.abandoned for c in calls) else
'RUNNING' if pending else 'IDLE', pending=pending,
pending_native=any(c.native_depth for c in calls))
def cancel(self):
self.stop.set()
def check_active(self):
if self.stop.is_set() or self._closed.is_set():
raise QueueCancelled('任务已取消,未提交新的原生操作。')
check_active()
def guard_callback(self, callback, *, current=None):
call = getattr(_local, 'call', None)
def guarded(*args, **kwargs):
# Cancellation still gets its own terminal result; closing or an
# abandoned invocation cannot update a newer task/library.
if (not self._closed.is_set() and not (call and call.abandoned)
and (current is None or current() is self)):
return callback(*args, **kwargs)
return guarded
def close(self, *, timeout_seconds=30):
self._closed.set()
self.cancel()
deadline = time.monotonic() + min(30, max(0, timeout_seconds))
with _tasks_guard:
calls = [c for c in _calls.values() if c.context is self]
for call in calls:
call.done.wait(max(0, deadline-time.monotonic()))
with _tasks_guard:
for call in calls:
if not call.done.is_set() and not call.native_depth:
call.abandoned = True
return self.snapshot()
def run(self, action, *, stop=None, timeout_seconds=10):
self.check_active()
if current_context() is self:
return action() # Preserve nested lease and mutex thread identity.
aliases = {_path_key(v) for v in (self.library, *self.aliases)}
call = _Call(self, stop)
with _tasks_guard:
if any(c.abandoned and aliases.intersection(
_path_key(v) for v in (c.context.library, *c.context.aliases))
for c in _calls.values()):
raise TaskHandoff(self)
_calls[id(call)] = call
def execute():
_local.call = call
try:
check_active()
call.result = action()
except BaseException as error:
call.error = error
finally:
with _tasks_guard:
call.done.set()
_calls.pop(id(call), None)
del _local.call
Thread(target=execute, name='cbb-task-'+self.operation_id[:12], daemon=True).start()
while not call.done.wait(.02):
with _tasks_guard:
if call.native_handoff:
cancelled = call.is_set()
call.abandoned = True
raise TaskHandoff(self, cancelled=cancelled, result=call.last_result)
elapsed = time.monotonic()-call.phase_started
if not call.native_depth and (call.abandoned or elapsed >= timeout_seconds):
cancelled = call.is_set()
call.abandoned = True
raise TaskHandoff(self, cancelled=cancelled, result=call.last_result)
if call.error is not None:
raise call.error
return call.result
class NativeQueue:
def __init__(self):
self.condition = Condition()
self.pending = []
self.active = False
self.closed = False
self.contexts = {}
def wake(self):
with self.condition: self.condition.notify_all()
def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None):
ticket = object()
def close(self, *, timeout_seconds=30):
deadline = time.monotonic()+min(30, max(0, timeout_seconds))
with self.condition:
self.closed = True
contexts = tuple(self.contexts.values())
for context in contexts:
context.cancel()
self.condition.notify_all()
return tuple(context.close(timeout_seconds=max(0, deadline-time.monotonic()))
for context in contexts)
def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None,
context=None, timeout_seconds=10):
ticket = [object(), False]
acquired, notified = False, False
previous_state = None
with self.condition: self.pending.append((ticket, ready))
with self.condition:
if self.closed:
raise QueueCancelled('队列已关闭,未启动新的操作。')
if context is not None:
self.contexts[id(context)] = context
self.pending.append(ticket)
try:
while not acquired:
available = ready() # Call only this ticket's predicate, outside the queue lock.
with self.condition:
if stop is not None and stop.is_set():
if (self.closed or stop is not None and stop.is_set()
or context is not None and context.stop.is_set()):
raise QueueCancelled('已取消排队,未启动原生操作。')
first = next((token for token, available in self.pending if available()), None)
ticket[1] = bool(available)
first = next((entry for entry in self.pending if entry[1]), None)
if not self.active and first is ticket:
self.pending = [entry for entry in self.pending if entry[0] is not ticket]
self.active = acquired = True
else:
self.condition.wait(.1)
if acquired and not ready():
with self.condition:
ticket[1] = self.active = acquired = False
self.condition.notify_all()
if not acquired and waiting is not None:
try:
current_state = wait_state() if wait_state is not None else None
@@ -41,9 +325,9 @@ class NativeQueue:
pass
if stop is not None and stop.is_set():
raise QueueCancelled('已取消排队,未启动原生操作。')
return action()
return context.run(action, stop=stop, timeout_seconds=timeout_seconds) if context else action()
finally:
with self.condition:
self.pending = [entry for entry in self.pending if entry[0] is not ticket]
self.pending = [entry for entry in self.pending if entry is not ticket]
if acquired: self.active = False
self.condition.notify_all()
+1 -1
View File
@@ -130,7 +130,7 @@ def register_worker_session(worker, job):
path = Path(job['session_record_path'])
from .icdb_ownership import session_resource_scope
write_session_record(path, {k: job.get(k) for k in (
'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal')}
'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal', 'worker_lease_handle')}
| {'processes': [{'role': 'worker', **process_identity(worker.pid)}],
'lease_state': 'ACTIVE', 'resource_scope': session_resource_scope(job),
'blocked_resources': [], 'data_recovery_required': None})
+102 -39
View File
@@ -545,14 +545,57 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, l
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None, version_prompt=None, stop=None):
# Cover contents checking, native work and final state reads with one
# session-wide mutex. The worker-stage lock below is reentrant on Windows.
import win32event, win32con
dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None,
admission_timeout_seconds=10):
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner,
port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory,
version_prompt=version_prompt)
bound = current_context()
if bound is None:
# Management may call Verify while holding a same-thread lease. Do not
# transfer that transaction to a different thread.
import threading
from .library_lock import _guard, _owners
with _guard:
nested = any(v['thread'] == threading.get_ident() and v['depth'] > 0
for v in _owners.values())
if not nested:
from .operation_queue import request_library_keys
bound = context or BatchTaskContext(library=str(request.lmc_file),
aliases=tuple(sorted(request_library_keys(request))), targets=(request.block_name,))
try:
return bound.run(lambda: run_verify(request, stop=stop, **options),
stop=stop, timeout_seconds=admission_timeout_seconds)
except TaskHandoff as error:
result = _handoff_verify_result(error)
_notify_progress(bound.guard_callback(progress) if progress else None,
dict(stage='task_handoff', operation_id=bound.operation_id,
library=bound.library, status=result.status, resource_scope=error.scope))
return result
if bound is not None and progress is not None:
options['progress'] = bound.guard_callback(progress)
return _run_verify_task(request, stop=task_stop(stop), **options)
def _handoff_verify_result(error):
known = error.result if isinstance(error.result, VerifyResult) else None
result = known or VerifyResult('CANCELLED' if error.cancelled else 'TIMEOUT')
return replace(result, status='FAIL' if result.status == 'PASS' else result.status,
stderr=result.stderr or str(error), safe_to_continue=False,
error_stage=result.error_stage or 'admission_or_finalization',
error_category=result.error_category or 'TASK_HANDOFF',
release_proof=dict(lease_state='PENDING', resource_scope=error.scope,
blocked_resources=error.scope.get('resources', [])))
def _run_verify_task(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
dialog_supervisor_factory=None, version_prompt=None, stop=None):
# Admission/finalization keep only the library lease. Native launch paths
# acquire the host channel after slow metadata has returned.
from .operation_queue import check_active
from .com_runtime import initialize_com, uninitialize_com
initialized = initialize_com()
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
library_lock = None
previous_release = None
release_confirmed = True
@@ -563,11 +606,7 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
from .library_lock import _supervisor_alive
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
try:
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
if not acquired:
return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。',
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')
check_active()
if request.maintenance:
from .maintenance_entry import run_icdb_check
return run_icdb_check(request, progress=progress, stop=stop)
@@ -576,10 +615,13 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
if Path(request.lmc_file).is_file():
try:
previous_release = recheck_previous_release(request.lmc_file,
Path(request.evidence_dir)/'previous-release', block_name=request.block_name, progress=progress)
Path(request.evidence_dir)/'previous-release', block_name=request.block_name,
progress=progress, stop=stop)
check_active()
library_lock = LibraryLock(request.lmc_file, checkpoint=False,
safe_to_release=clients_released)
library_lock.__enter__()
check_active()
except LibraryBusy as error:
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
except (OSError, ValueError, RuntimeError, pywintypes.error) as error:
@@ -691,14 +733,12 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
if library_lock is not None:
library_lock.__exit__(None, None, None)
finally:
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
uninitialize_com(initialized)
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
from .operation_queue import remember_result
timing = RunTimings()
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
process_runner=process_runner, port_check=port_check,
@@ -722,6 +762,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
except (OSError, ValueError, KeyError):
pass
remember_result(result)
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
job_path = result.evidence_path.with_name('request.json')
if job_path.is_file():
@@ -764,6 +805,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
if result.status not in {"PASS", "SKIPPED"} else ""),
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
if result.status not in {"PASS", "SKIPPED"} else ""))
remember_result(result)
if result.evidence_path:
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
@@ -819,7 +861,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
except Exception:
# Cleanup must never hide or change the verification result.
pass
return replace(result, evidence_path=result_path)
from .operation_queue import remember_result
return remember_result(replace(result, evidence_path=result_path))
if stop is not None and stop.is_set():
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
@@ -843,7 +886,9 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
'detail': ' '.join(command)})
timing.begin('content_check')
try:
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
from .operation_queue import native_channel
with native_channel():
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
finally:
timing.end('content_check')
combined = '\n'.join(part for part in (stdout, stderr) if part)
@@ -915,17 +960,15 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
from .desktop_worker import DesktopWorker
from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend
from .dialog_models import DialogPhase
import win32event, win32con
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
lock_status = win32event.WaitForSingleObject(mutex, 0)
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
mutex.Close()
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。',
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY'))
from .operation_queue import check_active, native_channel
check_active()
worker_launch_attempted = False
native = native_channel()
phase_entered = False
try:
timing.begin('worker_setup')
with (root/'worker.log').open('w', encoding='utf-8') as log:
check_active()
worker_launch_attempted = True
if request.interactive_input:
process = DesktopWorker(
@@ -965,6 +1008,13 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
)
if stop is not None and stop.is_set():
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
check_active()
try:
native.__enter__()
phase_entered = True
except RuntimeError as error:
return finish(VerifyResult('BLOCKED', stderr=str(error), safe_to_continue=False,
error_category='NATIVE_CHANNEL_BUSY'))
Path(job['start_gate']).touch()
while process.poll() is None:
if stop is not None and stop.is_set():
@@ -1188,6 +1238,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
{'stage': 'session_observation', 'block': request.block_name, **state}))
_notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name,
'detail': '已转后台观察,可查看状态'})
if getattr(process, 'native_write_pending', lambda: None)() is not False:
from .operation_queue import hold_native_handoff
hold_native_handoff(process, process.session_record_path)
worker_exited = process.poll() is not None
except Exception as error:
close_errors.append(str(error))
if 'process' in locals():
@@ -1279,8 +1333,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
except OSError:
pass
finally:
win32event.ReleaseMutex(mutex)
mutex.Close()
if phase_entered:
native.__exit__(None, None, None)
if result.status == 'PASS' and not request.diagnostic_only:
timing.begin('library_flag')
@@ -1290,9 +1344,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
if progress:
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
try:
probe = read_library_flag_with_retry(
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
request.block_name, on_attempt=record_attempt)
with native_channel():
probe = read_library_flag_with_retry(
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
request.block_name, on_attempt=record_attempt)
result = apply_post_verify_status(result, probe)
result = replace(result, bridge_attempts=tuple(bridge_records))
_write_optional_evidence(root/'bridge-status.json', bridge_records)
@@ -1304,20 +1359,27 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
return finish(result)
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None):
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None,
context=None):
"""Run serially and stop when database/session safety is unconfirmed."""
if max_workers != 1:
raise ValueError('Only serial Verify is supported for the shared Library Manager session')
if context is not None:
stop = stop if stop is not None else context.stop
progress = context.guard_callback(progress) if progress is not None else None
if verifier is None:
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available, stop=stop) # noqa: E731
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available,
stop=stop, context=context) # noqa: E731
results = []
queue_stop = False
retained_libraries = set()
for request in requests:
library_key = str(getattr(request, 'lmc_file', '__injected_verifier_scope__')).casefold()
if queue_stop or (stop is not None and stop.is_set()):
from .operation_queue import request_library_keys
library_keys = request_library_keys(request, context)
if (queue_stop or stop is not None and stop.is_set()
or context is not None and context.stop.is_set()):
result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB')
elif library_key in retained_libraries:
elif library_keys.intersection(retained_libraries):
result = VerifyResult('SKIPPED', stderr='同库原生会话尚未释放,未启动新的写入')
else:
try:
@@ -1325,18 +1387,19 @@ def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=Non
result = verifier(request)
if not result.safe_to_continue:
from .operation_lifecycle import recheck_failed_release
result = recheck_failed_release(request.lmc_file, request.block_name, result)
result = recheck_failed_release(request.lmc_file, request.block_name, result, stop=stop)
return result
result = item_executor(request, execute) if item_executor is not None else execute()
except Exception as error:
from .operation_queue import QueueCancelled
from .operation_queue import QueueCancelled, TaskHandoff
# An unexpected exception cannot establish database safety.
result = VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL',
stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled))
result = (_handoff_verify_result(error) if isinstance(error, TaskHandoff) else
VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL',
stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled)))
results.append(result)
# Never start another writer after an unconfirmed native shutdown.
if not result.safe_to_continue:
retained_libraries.add(library_key)
retained_libraries.update(library_keys)
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'}
if queue_stop and stop is not None:
stop.set()
+2 -2
View File
@@ -71,7 +71,7 @@ def test_verify_failure_recheck_finishes_before_yielding_native_queue(qtbot,tmp_
try:return action()
finally:active.pop()
return run
def recheck(library,block,result):
def recheck(library,block,result,**kwargs):
seen.append(bool(active))
return replace(result,safe_to_continue=True)
monkeypatch.setattr(window,'_queued_action',queued)
@@ -92,7 +92,7 @@ def test_management_failure_recheck_finishes_before_yielding_native_queue(tmp_pa
active.append(True)
try:return action()
finally:active.pop()
def recheck(library,block,result):
def recheck(library,block,result,**kwargs):
seen.append(bool(active))
return replace(result,safe_to_continue=True)
monkeypatch.setattr(operation_lifecycle,'recheck_failed_release',recheck)
+724
View File
@@ -0,0 +1,724 @@
"""Cancellation must release scheduling, never invent native release."""
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from threading import Event, Thread
import time
import pytest
import win32con
import win32event
from cbb_refresh_verify import library_lock as locks
from test_management import setup as management_setup
@pytest.mark.parametrize('stage', ['read', 'archive'])
def test_stalled_checkpoint_does_not_own_native_channel_or_commit_after_stop(tmp_path, monkeypatch, stage):
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease:
pass
marker = locks.blocker_path(library)
original = marker.read_bytes()
entered, resume, stop = Event(), Event(), Event()
read, write = Path.read_bytes, Path.write_bytes
def read_bytes(path):
if stage == 'read' and path == marker and not resume.is_set():
entered.set()
assert resume.wait(5)
return read(path)
def write_bytes(path, data):
if stage == 'archive' and path.name.startswith('blocker-before-'):
entered.set()
assert resume.wait(5)
return write(path, data)
monkeypatch.setattr(Path, 'read_bytes', read_bytes)
monkeypatch.setattr(Path, 'write_bytes', write_bytes)
monkeypatch.setattr('cbb_refresh_verify.owner_probe.query_owners', lambda *_: [])
with ThreadPoolExecutor(1) as pool:
task = pool.submit(locks.release_retained_library, library, tmp_path/'evidence', stop=stop)
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
try:
assert entered.wait(3)
acquired = win32event.WaitForSingleObject(mutex, 0) in (
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
assert acquired, 'checkpoint I/O retained the host native channel'
if stage == 'archive':
import json
receipt = next((tmp_path/'evidence').rglob('release.json'))
assert json.loads(receipt.read_text())['lease_state'] != 'RELEASED'
stop.set()
resume.set()
with pytest.raises(InterruptedError):
task.result(5)
assert read(marker) == original
assert locks.has_retained_library(library)
finally:
stop.set()
resume.set()
if acquired:
win32event.ReleaseMutex(mutex)
mutex.Close()
try:
task.result(5)
except InterruptedError:
pass
with locks._guard:
owner = locks._owners.pop(lease.key, None)
if owner:
owner['handle'].Close()
def test_task_context_is_frozen_pure_and_has_an_independent_monotonic_stop(tmp_path, monkeypatch):
from dataclasses import FrozenInstanceError
from cbb_refresh_verify.operation_queue import BatchTaskContext
monkeypatch.setattr(Path, 'resolve', lambda *a, **k: pytest.fail('context performed filesystem admission'))
first = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',), targets=('A',))
second = BatchTaskContext(library=str(tmp_path/'B.lmc'))
with pytest.raises(FrozenInstanceError):
first.library = second.library
first.cancel()
assert first.stop.is_set() and not second.stop.is_set()
assert not hasattr(first.stop, 'clear')
received = []
first.guard_callback(received.append)('CANCELLED')
assert received == ['CANCELLED']
first.close(timeout_seconds=0)
first.guard_callback(received.append)('late')
assert received == ['CANCELLED']
def test_bounded_queue_handoff_preserves_scope_and_fences_late_native_launch(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff, native_phase
queue = NativeQueue()
first = BatchTaskContext(library=str(tmp_path/'A.lmc'), targets=('A',))
second = BatchTaskContext(library=str(tmp_path/'B.lmc'), targets=('B',))
entered, resume, finished = Event(), Event(), Event()
effects = []
def stalled():
try:
entered.set()
assert resume.wait(3)
with native_phase():
effects.append('late native write')
finally:
finished.set()
try:
with pytest.raises(TaskHandoff) as error:
queue.run(stalled, context=first, timeout_seconds=.05)
assert entered.is_set() and error.value.library == first.library
assert first.snapshot()['pending'] == 1
assert queue.run(lambda: 'B ran', context=second) == 'B ran'
resume.set()
assert finished.wait(2)
assert effects == []
finally:
resume.set()
assert finished.wait(2)
def test_native_phase_is_never_yielded_at_metadata_deadline(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, native_phase
queue = NativeQueue()
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
entered, resume, finished = Event(), Event(), Event()
results = []
def transaction():
with native_phase():
entered.set()
assert resume.wait(3)
return 'Remove and Add completed'
def run():
results.append(queue.run(transaction, context=context, timeout_seconds=.02))
finished.set()
thread = Thread(target=run)
thread.start()
try:
assert entered.wait(2)
assert not finished.wait(.1)
state = context.close(timeout_seconds=.02)
assert state['pending_native'] and queue.active
resume.set()
assert finished.wait(2)
assert results == ['Remove and Add completed']
finally:
resume.set()
thread.join(2)
def test_stalled_ready_predicate_does_not_hold_queue_condition():
from cbb_refresh_verify.operation_queue import NativeQueue, QueueCancelled
queue = NativeQueue()
entered, resume, stop, completed = Event(), Event(), Event(), Event()
effects = []
def slow_ready():
entered.set()
assert resume.wait(3)
return True
def first():
try:
queue.run(lambda: effects.append('A'), ready=slow_ready, stop=stop)
except QueueCancelled:
pass
thread = Thread(target=first)
thread.start()
second = Thread(target=lambda: (queue.run(lambda: effects.append('B')), completed.set()))
try:
assert entered.wait(2)
second.start()
assert completed.wait(.5), 'one ready predicate froze every other queue ticket'
stop.set()
resume.set()
thread.join(2)
assert effects == ['B']
finally:
stop.set()
resume.set()
thread.join(3)
second.join(3)
def test_verify_admission_handoff_unblocks_b_and_never_launches_after_read_returns(tmp_path, monkeypatch):
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
from cbb_refresh_verify.verify_runner import VerifyRequest, run_verify
import win32process
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease:
pass
marker = locks.blocker_path(library)
entered, resume, returned = Event(), Event(), Event()
read = Path.read_text
def withheld(path, *args, **kwargs):
if path == marker:
entered.set()
assert resume.wait(3)
returned.set()
return read(path, *args, **kwargs)
monkeypatch.setattr(Path, 'read_text', withheld)
monkeypatch.setattr(win32process, 'CreateProcess', lambda *a, **k: pytest.fail('late native launch'))
context = BatchTaskContext(library=str(library), targets=('A',))
events = []
try:
result = run_verify(VerifyRequest(library, 'A', tmp_path/'evidence'), context=context,
admission_timeout_seconds=.05, progress=events.append)
assert entered.is_set() and result.status == 'TIMEOUT'
assert not result.safe_to_continue
assert result.release_proof['resource_scope']['operation_ids'] == [context.operation_id]
def unrelated():
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0
try:
assert acquired
return 'B completed'
finally:
if acquired: win32event.ReleaseMutex(mutex)
mutex.Close()
assert NativeQueue().run(unrelated) == 'B completed'
count = len(events)
resume.set()
assert returned.wait(2)
limit = time.monotonic()+2
while context.snapshot()['pending'] and time.monotonic() < limit:
time.sleep(.01)
assert context.snapshot()['pending'] == 0
assert len(events) == count
finally:
resume.set()
context.close(timeout_seconds=2)
with locks._guard:
owner = locks._owners.pop(lease.key, None)
if owner: owner['handle'].Close()
def test_native_time_does_not_consume_next_metadata_phase_budget(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext, native_phase
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
stages = []
def action():
with native_phase():
time.sleep(.12)
stages.append('saved')
time.sleep(.02)
with native_phase():
stages.append('closed')
return stages
assert context.run(action, timeout_seconds=.06) == ['saved', 'closed']
def test_native_channel_enforces_cross_thread_serialization_and_reentrancy():
from cbb_refresh_verify.operation_queue import native_channel
with native_channel():
with native_channel():
with ThreadPoolExecutor(1) as pool:
def contend():
with native_channel():
pytest.fail('concurrent native writer admitted')
with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'):
pool.submit(contend).result(2)
with native_channel():
pass
def test_maintenance_checkpoint_admission_does_not_hold_native_channel(tmp_path, monkeypatch):
from contextlib import nullcontext
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
with locks.LibraryLock(library, safe_to_release=lambda: False) as retained:
pass
marker = locks.blocker_path(library)
entered, resume = Event(), Event()
original = Path.read_bytes
def withheld(path):
if path == marker and not resume.is_set():
entered.set()
assert resume.wait(3)
return original(path)
monkeypatch.setattr(Path, 'read_bytes', withheld)
def admit():
with locks.LibraryLock.for_maintenance(library, tmp_path/'archive', lambda: nullcontext({})):
pass
with ThreadPoolExecutor(1) as pool:
task = pool.submit(admit)
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
acquired = False
try:
assert entered.wait(2)
acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0
assert acquired
finally:
if acquired: win32event.ReleaseMutex(mutex)
mutex.Close()
resume.set()
task.result(5)
with locks._guard:
owner = locks._owners.pop(retained.key, None)
if owner: owner['handle'].Close()
def test_native_handoff_keeps_real_mutex_until_bound_process_returns(tmp_path):
import subprocess
import sys
from cbb_refresh_verify.operation_queue import (
BatchTaskContext, NativeQueue, TaskHandoff, hold_native_handoff, native_channel)
from cbb_refresh_verify.session_record import write_session_record
process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
record = tmp_path/'session.json'
write_session_record(record, dict(operation_id=context.operation_id, library=context.library,
worker_pid=process.pid))
queue = NativeQueue()
effects = []
def action():
with native_channel():
hold_native_handoff(process, record)
with native_channel():
effects.append('late mutation')
try:
with pytest.raises(TaskHandoff):
queue.run(action, context=context)
assert process.poll() is None
assert context.snapshot()['pending_native']
assert queue.run(lambda: 'pure read') == 'pure read'
with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'):
with native_channel():
pytest.fail('writer overlapped handed-off native call')
process.communicate('\n', timeout=3)
limit = time.monotonic()+3
while context.snapshot()['pending'] and time.monotonic() < limit:
time.sleep(.01)
assert context.snapshot()['pending'] == 0
with native_channel():
pass
assert effects == []
finally:
if process.poll() is None:
process.communicate('\n', timeout=3)
context.close(timeout_seconds=3)
def test_worker_write_barrier_survives_supervisor_process_exit(tmp_path):
import json
import subprocess
import sys
from cbb_refresh_verify.operation_queue import native_channel
ready, finish, done = (tmp_path/name for name in ('ready', 'finish', 'done'))
child = '''import sys,time,os
from pathlib import Path
from cbb_refresh_verify.desktop_worker import worker_native_channel
with worker_native_channel():
Path(sys.argv[1]).write_text(str(os.getpid()))
while not Path(sys.argv[2]).exists(): time.sleep(.02)
Path(sys.argv[3]).touch()
'''
parent = '''import subprocess,sys,time,json
from pathlib import Path
subprocess.Popen([sys.executable,'-c',sys.argv[1],*sys.argv[2:]])
deadline=time.monotonic()+4
while not Path(sys.argv[2]).exists() and time.monotonic()<deadline: time.sleep(.02)
raise SystemExit(0 if Path(sys.argv[2]).exists() else 2)
'''
supervisor = subprocess.Popen([sys.executable, '-c', parent, child, str(ready), str(finish), str(done)])
try:
assert supervisor.wait(timeout=6) == 0
assert ready.is_file() and not done.exists()
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
with native_channel():
pytest.fail('restart admitted a second writer')
finally:
finish.touch()
deadline = time.monotonic()+3
while not done.exists() and time.monotonic()<deadline: time.sleep(.02)
supervisor.wait(timeout=3)
assert done.is_file()
with native_channel():
pass
def test_exact_sidecar_handle_survives_parent_exit_until_worker_boundary(tmp_path):
import subprocess
import sys
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
ready, finish, done, handle = (tmp_path/name for name in ('ready', 'finish', 'done', 'handle'))
child = '''import sys,time
from pathlib import Path
from cbb_refresh_verify.desktop_worker import worker_native_channel
while not Path(sys.argv[4]).exists(): time.sleep(.02)
with worker_native_channel(lease_handle=int(Path(sys.argv[4]).read_text())):
Path(sys.argv[1]).touch()
while not Path(sys.argv[2]).exists(): time.sleep(.02)
Path(sys.argv[3]).touch()
'''
parent = '''import subprocess,sys,time,win32api,win32con
from pathlib import Path
from cbb_refresh_verify.library_lock import LibraryLock,duplicate_worker_lease
with LibraryLock(sys.argv[1],checkpoint=False):
child=subprocess.Popen([sys.executable,'-c',sys.argv[2],*sys.argv[3:]])
process=win32api.OpenProcess(win32con.PROCESS_DUP_HANDLE,False,child.pid)
try: handle=duplicate_worker_lease(sys.argv[1],process)
finally: process.Close()
Path(sys.argv[6]).write_text(str(handle))
deadline=time.monotonic()+4
while not Path(sys.argv[3]).exists() and time.monotonic()<deadline: time.sleep(.02)
assert Path(sys.argv[3]).exists()
'''
supervisor = subprocess.Popen([sys.executable, '-c', parent, str(library), child,
str(ready), str(finish), str(done), str(handle)])
try:
assert supervisor.wait(timeout=6) == 0
with pytest.raises(locks.LibraryBusy):
with locks.LibraryLock(library, checkpoint=False):
pytest.fail('uploader could replace a live worker library')
finally:
finish.touch()
deadline = time.monotonic()+3
while ready.exists() and not done.exists() and time.monotonic()<deadline: time.sleep(.02)
supervisor.wait(timeout=3)
assert done.is_file()
with locks.LibraryLock(library, checkpoint=False):
pass
@pytest.mark.parametrize('obstacle', ['', 'unknown_title', 'duplicate', 'changed'])
def test_failed_startup_dismisses_only_exact_stable_owned_tip(tmp_path, monkeypatch, obstacle):
import win32gui
from test_failed_startup import startup_fixture
from cbb_refresh_verify.failed_session import close_failed_startup
job, records, messages = startup_fixture(tmp_path, monkeypatch)
visible = {100, 200}
reads = []
classes = {100: 'Afx:main', 200: '#32770', 201: 'Button', 202: 'Button'}
def title(hwnd):
if hwnd == 100: return 'Expedition PCB'
if hwnd == 200:
reads.append(hwnd)
return 'Unknown' if obstacle == 'unknown_title' or obstacle == 'changed' and len(reads)>1 else 'Tip of the Day'
return 'OK'
monkeypatch.setattr(win32gui, 'EnumWindows', lambda fn, arg: [fn(h, arg) for h in sorted(visible)])
monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda hwnd, fn, arg:
[fn(h, arg) for h in ((201, 202) if obstacle == 'duplicate' else (201,))] if hwnd == 200 else None)
monkeypatch.setattr(win32gui, 'GetClassName', classes.__getitem__)
monkeypatch.setattr(win32gui, 'GetWindowText', title)
monkeypatch.setattr(win32gui, 'GetDlgCtrlID', lambda hwnd: 1)
def post(hwnd, message, wparam, lparam):
messages.append((hwnd, message))
if hwnd == 201: visible.remove(200)
elif hwnd == 100: records.clear()
monkeypatch.setattr(win32gui, 'PostMessage', post)
result = close_failed_startup(job, 12, timeout_seconds=0)
if obstacle:
assert result['status'] == 'RETAINED' and not messages
else:
assert result['status'] == 'CLOSED'
assert messages == [(201, win32con.BM_CLICK), (100, win32con.WM_CLOSE)]
def test_late_release_recheck_is_bounded_and_keeps_business_failure(tmp_path, monkeypatch):
from cbb_refresh_verify.operation_lifecycle import recheck_failed_release
from cbb_refresh_verify.verify_runner import VerifyResult
from cbb_refresh_verify.session_record import write_session_record
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
evidence = tmp_path/'evidence'
evidence.mkdir()
result = VerifyResult('FAIL', stderr='native business failure', safe_to_continue=False,
evidence_path=evidence/'result.json')
result.evidence_path.write_text('{}')
record = evidence/'session.json'
write_session_record(record, dict(operation_id='failed', library=str(library), block='A', processes=[]))
entered, resume = Event(), Event()
original = Path.open
def withheld(path, *args, **kwargs):
if path == record:
entered.set()
assert resume.wait(3)
return original(path, *args, **kwargs)
monkeypatch.setattr(Path, 'open', withheld)
try:
started = time.monotonic()
returned = recheck_failed_release(library, 'A', result, timeout_seconds=.05)
assert time.monotonic()-started < .8
assert entered.is_set() and returned is result
assert returned.status == 'FAIL' and not returned.safe_to_continue
finally:
resume.set()
def test_batch_context_cancel_records_all_remaining_items_and_new_batch_can_run(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext
from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch
requests = [VerifyRequest(tmp_path/'lib.lmc', name, tmp_path/'evidence') for name in ('A', 'B', 'C')]
first = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A', 'B', 'C'))
events = []
def verify(request):
first.cancel()
return VerifyResult('CANCELLED')
results = run_batch(requests, verifier=verify, context=first,
progress=lambda name, result: events.append((name, result.status)))
assert [r.status for r in results] == ['CANCELLED', 'SKIPPED', 'SKIPPED']
assert events == [('A', 'CANCELLED'), ('B', 'SKIPPED'), ('C', 'SKIPPED')]
second = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A',))
assert run_batch(requests[:1], verifier=lambda _: VerifyResult('PASS'), context=second)[0].status == 'PASS'
def test_batch_retention_matches_normalized_library_alias_not_business_failure(tmp_path):
from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch
library = tmp_path/'A/lib.lmc'
alias = tmp_path/'A/sub/../lib.lmc'
requests = [VerifyRequest(path, block, tmp_path/'evidence') for path, block in (
(library, 'A'), (alias, 'B'), (tmp_path/'B/lib.lmc', 'C'), (tmp_path/'B/lib.lmc', 'D'))]
calls = []
def verify(request):
calls.append(request.block_name)
return VerifyResult('FAIL', safe_to_continue=request.block_name != 'A')
results = run_batch(requests, verifier=verify)
assert calls == ['A', 'C', 'D']
assert [r.status for r in results] == ['FAIL', 'SKIPPED', 'FAIL', 'FAIL']
def test_queue_close_has_one_deadline_and_reports_pending_scope(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff
queue = NativeQueue()
context = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',))
entered, resume, finished = Event(), Event(), Event()
def action():
entered.set()
resume.wait(3)
def run():
try:
queue.run(action, context=context)
except TaskHandoff:
pass
finally:
finished.set()
thread = Thread(target=run)
thread.start()
try:
assert entered.wait(2)
started = time.monotonic()
states = queue.close(timeout_seconds=.05)
assert time.monotonic()-started < .5
assert states[0]['resource_scope']['aliases'] == ['Z:/A.lmc']
assert states[0]['pending'] == 1
assert finished.wait(1)
finally:
resume.set()
thread.join(2)
def test_real_management_transaction_does_not_handoff_between_remove_and_add(management_setup):
from dataclasses import replace
from test_management import execute
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
request, adapter, source = management_setup
original = adapter.remove
def slow_remove(*args):
result = original(*args)
time.sleep(.4)
return result
adapter.remove = slow_remove # Native external boundary; transaction stays real.
context = BatchTaskContext(library=str(request.library), targets=('Block',))
result = NativeQueue().run(lambda: execute(replace(request, operation='update',
source=source, auto_verify=False), adapter), context=context, timeout_seconds=.2)
assert result.status == 'UPLOADED'
assert adapter.calls[0] == 'remove' and adapter.calls[1][0] == 'add'
def test_busy_native_channel_before_dispatch_does_not_retain_management_library(management_setup):
from test_management import execute
from cbb_refresh_verify.operation_queue import native_channel
request, adapter, _ = management_setup
entered, resume = Event(), Event()
def owner():
with native_channel():
entered.set()
resume.wait(3)
thread = Thread(target=owner)
thread.start()
try:
assert entered.wait(2)
result = execute(request, adapter)
assert result.safe_to_continue and not adapter.calls
with locks.LibraryLock(request.library):
pass
finally:
resume.set()
thread.join(3)
def test_only_explicit_repair_inspection_can_enter_while_worker_owns_writer_barrier(tmp_path):
import json
import os
from cbb_refresh_verify.desktop_worker import worker_native_channel, run_native_worker
from cbb_refresh_verify.operation_queue import native_channel
from cbb_refresh_verify.native_repair_ui import repair_worker_main
entered, resume = Event(), Event()
def owner():
with worker_native_channel():
entered.set()
resume.wait(3)
thread = Thread(target=owner)
thread.start()
job = dict(root=str(tmp_path), project=str(tmp_path/'A.prj'), database=str(tmp_path/'database'),
cancel_signal=str(tmp_path/'cancel'), supervisor_pid=os.getpid(), desktop='not-private')
path = tmp_path/'repair-request.json'
try:
assert entered.wait(2)
with native_channel(inspection_only=True):
pass
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
with native_channel():
pass
for mode in (False, 'true'):
path.write_text(json.dumps({**job, 'inspection_only': mode}))
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
run_native_worker(repair_worker_main, path)
path.write_text(json.dumps({**job, 'inspection_only': True}))
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
run_native_worker(lambda _: pytest.fail('arbitrary worker exempted'), path)
assert run_native_worker(repair_worker_main, path) == 1
result = json.loads((tmp_path/'repair-result.json').read_text())
assert result['native_started'] is False
assert 'REPAIR_PRIVATE_DESKTOP_REQUIRED' in result['detail']
finally:
resume.set()
thread.join(3)
def test_inspection_session_does_not_take_over_retained_writer_lease(tmp_path):
import os
import win32api
from cbb_refresh_verify.desktop_worker import DesktopWorker
library = tmp_path/'A/lib.lmc'
library.parent.mkdir()
library.touch()
with locks.LibraryLock(library, safe_to_release=lambda: False) as lease:
pass
worker = object.__new__(DesktopWorker)
worker.handle, worker.pid = win32api.GetCurrentProcess(), os.getpid()
try:
worker.bind_session(dict(operation_id='inspection', operation='inspect', inspection_only=True,
library=str(library), project=str(tmp_path/'A.prj'), session_record_path=str(tmp_path/'session.json')))
assert locks.has_retained_library(library)
from cbb_refresh_verify.session_record import read_session_record
record = read_session_record(tmp_path/'session.json', evidence_root=tmp_path)
assert record['worker_lease_handle'] is None
finally:
with locks._guard:
owner = locks._owners.pop(lease.key, None)
if owner: owner['handle'].Close()
def test_handoff_preserves_known_business_result_and_save_facts(tmp_path):
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, remember_result
from cbb_refresh_verify.verify_runner import VerifyResult
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
native_result = VerifyResult('FAIL', stderr='native failure after Save', safe_to_continue=False,
write_outcome=dict(save_state='COMMITTED', close_state='UNKNOWN'))
resume, done = Event(), Event()
def finalize():
remember_result(native_result)
try:
resume.wait(3)
finally:
done.set()
try:
with pytest.raises(TaskHandoff) as error:
context.run(finalize, timeout_seconds=.05)
assert error.value.result is native_result
assert error.value.result.write_outcome['save_state'] == 'COMMITTED'
assert error.value.result.status == 'FAIL'
from cbb_refresh_verify.verify_runner import _handoff_verify_result
returned = _handoff_verify_result(error.value)
assert returned.status == 'FAIL' and returned.stderr == native_result.stderr
assert returned.write_outcome == native_result.write_outcome and not returned.safe_to_continue
error.value.result = VerifyResult('PASS', write_outcome=native_result.write_outcome)
unfinished = _handoff_verify_result(error.value)
assert unfinished.status == 'FAIL' and unfinished.library_flag == 'UNKNOWN'
assert not unfinished.safe_to_continue
finally:
resume.set()
assert done.wait(2)
def test_registered_writer_owns_process_handle_after_callers_handle_closes(monkeypatch):
import subprocess
import sys
from cbb_refresh_verify import desktop_worker
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
entered, done = Event(), Event()
errors = []
def run():
try:
with desktop_worker.worker_native_channel():
desktop_worker.register_native_writer(process)
process._handle.Close()
entered.set()
except Exception as error:
errors.append(error)
finally:
done.set()
thread = Thread(target=run)
thread.start()
try:
assert entered.wait(2)
assert not done.wait(.1), 'a closed caller handle released an unreturned writer'
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
desktop_worker.assert_native_writer_idle()
finally:
process.stdin.write('\n')
process.stdin.flush()
process.stdin.close()
thread.join(3)
# The duplicate's observed exit is authoritative; the original Popen
# handle was deliberately invalidated, so do not poll it in __del__.
process.returncode = 0
assert done.is_set() and not errors
+3 -1
View File
@@ -17,7 +17,9 @@ def test_batch_rechecks_owned_failure_before_skipping_next_item(tmp_path,monkeyp
monkeypatch.setattr(session_record,'inspect_session_record',lambda *a,**k:dict(status='UNCONFIRMED'))
def proof(*a,**k):
if not released:raise library_lock.LibraryBusy('shared database still occupied')
return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False)
return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False,
lease_state='RELEASED',blocked_resources=[],
resource_scope=dict(scope_state='OBSERVED',library=str(library)))
monkeypatch.setattr(library_lock,'release_retained_library',proof)
def verify(req):
calls.append(req.block_name)
+1 -1
View File
@@ -215,7 +215,7 @@ def test_cancel_and_restart_never_replay(setup):
from cbb_refresh_verify.management import run_management_batch, unfinished_operations
request, adapter, _ = setup
stop = Event(); stop.set()
assert run_management_batch([request], stop=stop, adapter=adapter) == []
assert [r.status for r in run_management_batch([request], stop=stop, adapter=adapter)] == ['SKIPPED']
checkpoint = request.evidence_dir/'interrupted/checkpoint.json'
checkpoint.parent.mkdir(parents=True)
checkpoint.write_text('{"stage":"native_remove", "status":"RUNNING"}', encoding='utf-8')
+4
View File
@@ -313,8 +313,12 @@ def test_private_worker_timeout_retains_native_and_never_kills(tmp_path, monkeyp
def test_hidden_worker_and_probe_dispatch_before_qt(monkeypatch, tmp_path):
import cbb_refresh_verify.native_management as native
from cbb_refresh_verify.application import main
from cbb_refresh_verify import desktop_worker
# This dispatch represents a new worker process, not other tests' mocked clients.
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
seen = []
monkeypatch.setattr(native, 'management_worker_main', lambda p: (seen.append(p) or 17))
(tmp_path/'job.json').write_text('{}')
assert main(['--management-worker', str(tmp_path/'job.json')]) == 17
assert seen == [tmp_path/'job.json']
+2 -2
View File
@@ -94,7 +94,7 @@ def test_user_can_close_unknown_client_before_confirming(tmp_path,monkeypatch):
_idle_check(*paths,confirmation=WriteAdmission(prompt=prompt),evidence=tmp_path/'evidence',block='Block')
@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED'])])
@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED','SKIPPED'])])
def test_batch_continues_only_after_skip_or_confirmed_release(tmp_path,monkeypatch,first,expected):
from dataclasses import replace
from test_management import FileAdapter,project
@@ -149,7 +149,7 @@ def test_cancel_batch_also_stops_default_api_without_a_caller_event(tmp_path,mon
request=ManagementRequest('add',library,'One',source=source,auto_verify=False,evidence_dir=tmp_path/'evidence')
adapter=FileAdapter()
results=run_management_batch([request,replace(request,name='Two')],adapter=adapter,version_prompt=prompt)
assert [r.status for r in results]==['CANCELLED']
assert [r.status for r in results]==['CANCELLED','SKIPPED']
assert len(choices)==1 and not adapter.calls