fix: bound task handoff and preserve native writer ownership
This commit is contained in:
1 parent
0440b2efbf
commit
ddff6bd857
18 files changed
+1397
-118
No files matched your search
@@ -98,13 +98,16 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
return owner_probe_main(args.owner_probe)
|
||||
if args.native_backup_worker:
|
||||
from .native_backup import native_backup_worker_main
|
||||
return native_backup_worker_main(args.native_backup_worker)
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(native_backup_worker_main, args.native_backup_worker)
|
||||
if args.management_worker:
|
||||
from .native_management import management_worker_main
|
||||
return management_worker_main(args.management_worker)
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(management_worker_main, args.management_worker)
|
||||
if args.repair_worker:
|
||||
from .native_repair_ui import repair_worker_main
|
||||
return repair_worker_main(args.repair_worker)
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(repair_worker_main, args.repair_worker)
|
||||
if args.collect_failure_logs:
|
||||
from .verify_diagnostics import failure_logs_main
|
||||
return failure_logs_main(args.collect_failure_logs)
|
||||
@@ -116,7 +119,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
import contextlib
|
||||
with args.verify_worker.with_name('worker.log').open('a', encoding='utf-8') as log:
|
||||
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log):
|
||||
return worker_main(args.verify_worker)
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(worker_main, args.verify_worker)
|
||||
context = configure(resolve_data_root(args.data_root)); logger = context.logger; logger.info("application_start")
|
||||
try:
|
||||
if args.batch_lmc:
|
||||
|
||||
@@ -16,6 +16,110 @@ _automation_clients = []
|
||||
_automation_clients_lock = threading.Lock()
|
||||
_supervisor_lost = threading.Event()
|
||||
_native_phases = {}
|
||||
WRITER_MUTEX = r'Local\CbbRefreshVerify795NativeWriter'
|
||||
|
||||
|
||||
def assert_native_writer_idle():
|
||||
mutex = win32event.CreateMutex(None, False, WRITER_MUTEX)
|
||||
acquired = False
|
||||
try:
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回,未启动新的写入。')
|
||||
finally:
|
||||
if acquired: win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
|
||||
|
||||
class _RegisteredWriter:
|
||||
def __init__(self, handle):
|
||||
import win32api
|
||||
current = win32api.GetCurrentProcess()
|
||||
self.handle = win32api.DuplicateHandle(current, handle, current, 0, False,
|
||||
win32con.DUPLICATE_SAME_ACCESS)
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
if self.returncode is None and win32event.WaitForSingleObject(self.handle, 0) == win32con.WAIT_OBJECT_0:
|
||||
self.returncode = win32process.GetExitCodeProcess(self.handle)
|
||||
self.handle.Close()
|
||||
return self.returncode
|
||||
|
||||
|
||||
def register_native_writer(process):
|
||||
handle = getattr(process, 'handle', None) or getattr(process, '_handle', None)
|
||||
registered = _RegisteredWriter(handle) if handle is not None else process
|
||||
with _automation_clients_lock:
|
||||
_automation_clients.append(registered)
|
||||
return process
|
||||
|
||||
|
||||
@contextmanager
|
||||
def worker_native_channel(*, lease_handle=None, inspection_only=False):
|
||||
"""The native worker, not its disposable supervisor, owns write exclusion."""
|
||||
import win32api
|
||||
mutex = win32event.CreateMutex(None, False, WRITER_MUTEX)
|
||||
acquired = False
|
||||
try:
|
||||
if inspection_only is not True:
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
raise RuntimeError('NATIVE_WRITER_ACTIVE: 原生事务尚未返回。')
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
# Only registered Save/Close/Repair clients extend the boundary;
|
||||
# an idle PCB shell or a business failure is not a pending write.
|
||||
warned = False
|
||||
while True:
|
||||
with _automation_clients_lock:
|
||||
clients = tuple(_automation_clients)
|
||||
try:
|
||||
released = all(client.poll() is not None for client in clients)
|
||||
except Exception:
|
||||
released = False
|
||||
if not warned:
|
||||
logging.getLogger(__name__).warning('Native writer exit is unconfirmed', exc_info=True)
|
||||
warned = True
|
||||
if released:
|
||||
break
|
||||
threading.Event().wait(.1)
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
if lease_handle is not None:
|
||||
win32api.CloseHandle(int(lease_handle))
|
||||
|
||||
|
||||
def run_native_worker(action, request_path):
|
||||
"""Wait for durable parent admission before acquiring the worker barrier."""
|
||||
import json
|
||||
import time
|
||||
job = json.loads(Path(request_path).read_text(encoding='utf-8'))
|
||||
gate = Path(job['start_gate']) if job.get('start_gate') else None
|
||||
if gate is not None:
|
||||
deadline = time.monotonic()+30
|
||||
while not gate.is_file():
|
||||
if (job.get('cancel_signal') and Path(job['cancel_signal']).exists()
|
||||
or time.monotonic() >= deadline):
|
||||
raise RuntimeError('WORKER_CANCELLED: 未取得启动许可。')
|
||||
threading.Event().wait(.05)
|
||||
lease_handle = None
|
||||
if job.get('session_record_path'):
|
||||
from .session_record import read_session_record
|
||||
path = Path(job['session_record_path'])
|
||||
record = read_session_record(path, evidence_root=path.parent)
|
||||
if (record['operation_id'] != job['operation_id']
|
||||
or not any(p['pid'] == os.getpid() for p in record.get('processes', ()) )):
|
||||
raise RuntimeError('WORKER_ADMISSION_IDENTITY_CHANGED')
|
||||
lease_handle = record.get('worker_lease_handle')
|
||||
inspection = (action.__module__ == 'cbb_refresh_verify.native_repair_ui'
|
||||
and action.__name__ == 'repair_worker_main' and job.get('inspection_only') is True)
|
||||
with worker_native_channel(lease_handle=lease_handle, inspection_only=inspection):
|
||||
if job.get('cancel_signal') and Path(job['cancel_signal']).exists():
|
||||
raise RuntimeError('WORKER_CANCELLED: 已取消原生启动。')
|
||||
return action(request_path)
|
||||
|
||||
|
||||
def _phase_pending(phase):
|
||||
@@ -56,6 +160,8 @@ def _record_write_pending(record_path, operation_id=None):
|
||||
def launch_save_helper(command, *, start_gate=None, identity_path=None, operation_id=None, **kwargs):
|
||||
"""Register the save client atomically with respect to supervisor loss."""
|
||||
with _automation_clients_lock:
|
||||
from .operation_queue import check_active
|
||||
check_active()
|
||||
if _supervisor_lost.is_set():
|
||||
raise RuntimeError('VERIFY_CANCELLED: 监督器已退出,未启动新的保存客户端')
|
||||
helper = subprocess.Popen(command, **kwargs)
|
||||
@@ -160,6 +266,8 @@ class DesktopWorker:
|
||||
startup = win32process.STARTUPINFO()
|
||||
# subprocess.STARTUPINFO does NOT pass lpDesktop to CreateProcess.
|
||||
startup.lpDesktop = startup_desktop_name(name, interactive_input=self.interactive_input)
|
||||
from .operation_queue import check_active
|
||||
check_active()
|
||||
self.handle, thread, self.pid, _ = win32process.CreateProcess(
|
||||
command[0], subprocess.list2cmdline(command), None, None, False,
|
||||
win32con.CREATE_NO_WINDOW, None, None, startup)
|
||||
@@ -199,6 +307,8 @@ class DesktopWorker:
|
||||
raise
|
||||
|
||||
def start_companion(self, command):
|
||||
from .operation_queue import check_active
|
||||
check_active()
|
||||
startup = win32process.STARTUPINFO()
|
||||
startup.lpDesktop = startup_desktop_name(
|
||||
self.name, interactive_input=getattr(self, 'interactive_input', False))
|
||||
@@ -283,7 +393,10 @@ class DesktopWorker:
|
||||
|
||||
def bind_session(self, job):
|
||||
from .session_record import register_worker_session
|
||||
register_worker_session(self, job)
|
||||
from .library_lock import duplicate_worker_lease
|
||||
lease = (None if job.get('inspection_only') is True else
|
||||
duplicate_worker_lease(job['library'], self.handle))
|
||||
register_worker_session(self, {**job, 'worker_lease_handle': lease})
|
||||
|
||||
def native_write_pending(self):
|
||||
return _record_write_pending(getattr(self, 'session_record_path', None))
|
||||
|
||||
@@ -53,6 +53,14 @@ def _cancel_target(snapshot):
|
||||
return None
|
||||
|
||||
|
||||
def _tip_target(snapshot):
|
||||
from .dialog_agent import DialogActionExecutor
|
||||
if snapshot.title != 'Tip of the Day' or snapshot.window_class != '#32770':
|
||||
return None
|
||||
target = DialogActionExecutor._target(snapshot, {'control_id': 1})
|
||||
return target if target is not None and target.class_name == 'Button' else None
|
||||
|
||||
|
||||
def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event: None,
|
||||
timeout_seconds=20, launcher_process=None):
|
||||
"""Normally close our empty PCB after a diagnosed database startup failure.
|
||||
@@ -84,6 +92,7 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event:
|
||||
declined_hwnd = None
|
||||
declined_kind = None
|
||||
cancelled_open = False
|
||||
dismissed_tips = set()
|
||||
while True:
|
||||
records = [r for r in _process_details('ExpeditionPCB.exe') if r[0] == pcb_pid]
|
||||
if not records:
|
||||
@@ -121,6 +130,24 @@ def close_failed_startup(job, pcb_pid, *, baseline_pids=(), emit=lambda **event:
|
||||
if declined_hwnd is not None and declined_hwnd not in windows:
|
||||
cancelled_open = True
|
||||
result['cancelled_prompt'] = declined_kind
|
||||
dismissed = False
|
||||
for hwnd in windows:
|
||||
if hwnd in dismissed_tips or win32gui.GetClassName(hwnd) != '#32770':
|
||||
continue
|
||||
snapshot = _startup_snapshot(hwnd, pcb_pid)
|
||||
target = _tip_target(snapshot)
|
||||
if target is None:
|
||||
continue
|
||||
current = _startup_snapshot(hwnd, pcb_pid)
|
||||
if current != snapshot or _tip_target(current) != target:
|
||||
raise ValueError('STARTUP_TIP_CHANGED')
|
||||
win32gui.PostMessage(target.handle, win32con.BM_CLICK, 0, 0)
|
||||
dismissed_tips.add(hwnd)
|
||||
dismissed = True
|
||||
emit(stage='startup_tip_dismissed', pcb_pid=pcb_pid, hwnd=hwnd)
|
||||
break
|
||||
if dismissed:
|
||||
continue # Prove the Tip disappeared; it does not prove a cancelled open.
|
||||
if declined_hwnd is None:
|
||||
for hwnd in windows:
|
||||
if win32gui.GetClassName(hwnd) != '#32770':
|
||||
|
||||
@@ -20,6 +20,21 @@ def _library_guard(key):
|
||||
return _library_guards.setdefault(key, threading.RLock())
|
||||
|
||||
|
||||
def duplicate_worker_lease(library, process_handle):
|
||||
"""Keep the exact admitted exclusive file object alive across parent exit."""
|
||||
import win32api, win32con
|
||||
lease = LibraryLock(library)
|
||||
with _library_guard(lease.key):
|
||||
owner = _owners.get(lease.key)
|
||||
if owner is None:
|
||||
return None
|
||||
if owner['thread'] != threading.get_ident() or owner['retained'] or not owner['depth']:
|
||||
raise LibraryBusy('不能将其他任务的库锁交给工作进程。')
|
||||
remote = win32api.DuplicateHandle(win32api.GetCurrentProcess(), owner['handle'],
|
||||
process_handle, 0, False, win32con.DUPLICATE_SAME_ACCESS)
|
||||
return remote.Detach() # This handle belongs to the target process only.
|
||||
|
||||
|
||||
class LibraryBusy(OSError):
|
||||
pass
|
||||
|
||||
@@ -407,14 +422,12 @@ def _owner_is_blocking(item, library, roots):
|
||||
|
||||
def _enter_maintenance(lease):
|
||||
"""Transfer only a valid known lease, after project admission under lock."""
|
||||
import win32con, win32event, win32file
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
import win32con, win32file
|
||||
from .operation_queue import check_active
|
||||
handle = None
|
||||
existing = None
|
||||
try:
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired: raise LibraryBusy('已有 CBB 操作运行,维护不能接续')
|
||||
check_active()
|
||||
with _library_guard(lease.key):
|
||||
marker = blocker_path(lease.library)
|
||||
if marker.with_suffix('.pending').exists():
|
||||
@@ -449,6 +462,7 @@ def _enter_maintenance(lease):
|
||||
with saved.open('xb') as stream:
|
||||
stream.write(old); stream.flush(); os.fsync(stream.fileno())
|
||||
if saved.read_bytes() != old: raise OSError('原检查点归档校验失败')
|
||||
check_active()
|
||||
lease_id = uuid4().hex
|
||||
_persist_blocker(lease.library, 'OPERATION_IN_PROGRESS', lease_id=lease_id)
|
||||
_owners[lease.key] = {'thread': threading.get_ident(), 'depth': 1, 'handle': handle,
|
||||
@@ -456,12 +470,9 @@ def _enter_maintenance(lease):
|
||||
'release_probe': lease.safe_to_release, 'block_name': None}
|
||||
lease.maintenance_proof = proof
|
||||
lease.entered = True
|
||||
lease.maintenance_mutex = mutex
|
||||
return lease
|
||||
except Exception:
|
||||
if handle is not None and existing is None: handle.Close()
|
||||
if acquired: win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
raise
|
||||
|
||||
|
||||
@@ -550,7 +561,13 @@ def _prove_library_released(library, block_name=None, *, block_names=(), allow_u
|
||||
# and compare the exact address/project binding before commit.
|
||||
stack.close()
|
||||
from .icdb_ownership import inspect_release_addresses
|
||||
inspections = inspect_release_addresses(files, stop=stop)
|
||||
# Checkpoint I/O is per-library; only native inspection owns
|
||||
# the host channel. The sidecar still excludes other writers.
|
||||
from .operation_queue import native_channel
|
||||
with native_channel(inspection_only=True):
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('已停止释放检查')
|
||||
inspections = inspect_release_addresses(files, stop=stop)
|
||||
try:
|
||||
current = stack.enter_context(hold_release_files(library, roots, stop=stop))
|
||||
except OSError as error:
|
||||
@@ -582,7 +599,7 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
|
||||
The upload sidecar is never deleted. This acknowledges process release,
|
||||
not the correctness of a partially changed library or its recovery state.
|
||||
"""
|
||||
import win32con, win32event, win32file
|
||||
import win32con, win32file
|
||||
library = Path(library).resolve(strict=True)
|
||||
from .session_record import (retained_session_snapshot, acknowledge_library_release,
|
||||
reconciled_retained_sessions)
|
||||
@@ -594,15 +611,10 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
|
||||
if evidence.is_relative_to(library.parent) or library.parent.is_relative_to(evidence):
|
||||
raise ValueError('释放检查证据必须位于目标库之外。')
|
||||
lease = LibraryLock(library)
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
external = None
|
||||
try:
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('已停止释放检查')
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
raise LibraryBusy('另一个 CBB 操作仍在运行。')
|
||||
with _library_guard(lease.key):
|
||||
owner = _owners.get(lease.key)
|
||||
if owner and (not owner['retained'] or owner['depth']):
|
||||
@@ -675,7 +687,8 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
|
||||
'block_names': list(scoped_blocks),
|
||||
'lease_id': owner['lease_id'] if owner else None,
|
||||
'operation_ids': [item['operation_id'] for item in reconciled_sessions]}
|
||||
(root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
pending = {**record, 'status': 'RELEASE_PENDING', 'lease_state': 'PENDING'}
|
||||
(root/'release.json').write_text(json.dumps(pending, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
for index, checkpoint in enumerate(checkpoints):
|
||||
if checkpoint.exists():
|
||||
data = checkpoint.read_bytes()
|
||||
@@ -683,15 +696,15 @@ def release_retained_library(library, evidence_dir, *, block_name=None, block_na
|
||||
archived.write_bytes(data)
|
||||
if archived.read_bytes() != data:
|
||||
raise OSError('阻止记录归档校验失败,保持库锁。')
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('已停止释放检查')
|
||||
checkpoint.unlink() # Checkpoint JSON only; never .lock.
|
||||
if owner:
|
||||
owner['handle'].Close()
|
||||
del _owners[lease.key]
|
||||
(root/'release.json').write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
acknowledge_library_release(record, matching_retained, block_name=scoped)
|
||||
return record
|
||||
finally:
|
||||
if external is not None:
|
||||
external.Close()
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
@@ -173,7 +173,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
write_admission=None):
|
||||
from .verify_runner import VerifyRequest, run_verify
|
||||
from .backup_recovery import _idle_check
|
||||
import win32con, win32event
|
||||
from contextlib import ExitStack
|
||||
from .operation_queue import check_active, native_channel, task_stop
|
||||
stop = task_stop(stop)
|
||||
import pythoncom
|
||||
if adapter is None:
|
||||
from .native_management import NativeManagementAdapter
|
||||
@@ -190,8 +192,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
history = []
|
||||
mutated = uploaded = removed = False
|
||||
owner_safe = True
|
||||
mutex = None
|
||||
acquired = False
|
||||
release_proof = None
|
||||
original = None
|
||||
change_report = None
|
||||
@@ -199,12 +199,13 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
nonlocal owner_safe
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('MANAGEMENT_CANCELLED: 已停止后续原生操作')
|
||||
owner_safe = False
|
||||
try:
|
||||
result = callback(*args)
|
||||
except Exception as error:
|
||||
owner_safe = getattr(error, 'safe_to_continue', False) is True
|
||||
raise
|
||||
with native_channel():
|
||||
owner_safe = False
|
||||
try:
|
||||
result = callback(*args)
|
||||
except Exception as error:
|
||||
owner_safe = getattr(error, 'safe_to_continue', False) is True
|
||||
raise
|
||||
owner_safe = result.safe_to_continue
|
||||
return result
|
||||
def checkpoint(next_stage, **details):
|
||||
@@ -292,14 +293,10 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
report = root/'checkpoint.json'
|
||||
checkpoint('release_check')
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
return finish('BLOCKED', '已有 CBB 校验/管理/恢复任务运行;本次未启动。')
|
||||
from .library_lock import recheck_previous_release
|
||||
release_proof = recheck_previous_release(library, root/'previous-release', block_name=request.name,
|
||||
required=request.check_previous_release or bool(unfinished_target
|
||||
and unfinished_target.get('safe_to_continue') is not True), progress=progress)
|
||||
and unfinished_target.get('safe_to_continue') is not True), progress=progress, stop=stop)
|
||||
if release_proof is not None:
|
||||
checkpoint('release_check', release_proof=release_proof)
|
||||
if unfinished_target:
|
||||
@@ -308,8 +305,9 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
adapter.bind_library_mapping(request.mapped_drive, request.mapped_remote_root)
|
||||
if hasattr(adapter,'bind_write_admission'):
|
||||
adapter.bind_write_admission(write_admission if native_idle_check else None)
|
||||
check_active()
|
||||
with LibraryLock(library, block_name=request.name, safe_to_release=lambda: owner_safe or (
|
||||
callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))):
|
||||
callable(getattr(adapter, 'release_confirmed', None)) and adapter.release_confirmed(library))), ExitStack() as native_stack:
|
||||
work = library.parent/'ReusableBlocksLP'/request.name/'Work'
|
||||
target_project, target_pcb = work/(request.name+'.prj'), work/'PCB/pcb.pcb'
|
||||
if request.operation != 'add':
|
||||
@@ -324,7 +322,6 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
if source:
|
||||
idle_check(library, source, pcb)
|
||||
checkpoint('catalog_before')
|
||||
owner_safe = False
|
||||
before = invoke(adapter.catalog, library, root/'before')
|
||||
owner_safe = before.safe_to_continue
|
||||
if not owner_safe or Path(before.library).resolve() != canonical_library or before.flow != 1:
|
||||
@@ -366,6 +363,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
if request.operation in ('remove', 'update'):
|
||||
if stop is not None and stop.is_set():
|
||||
return finish('CANCELLED', '已取消,未开始移除', owner_safe)
|
||||
native_stack.enter_context(native_channel())
|
||||
checkpoint('native_remove')
|
||||
mutated = True; owner_safe = False
|
||||
native = invoke(adapter.remove, request, root/'remove')
|
||||
@@ -385,6 +383,8 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
if stop is not None and stop.is_set():
|
||||
return finish('RECOVERY_REQUIRED' if mutated else 'CANCELLED',
|
||||
'已停止后续导入;已发生的原生变更需要核对', owner_safe)
|
||||
if not removed:
|
||||
native_stack.enter_context(native_channel())
|
||||
checkpoint('native_add')
|
||||
mutated = True; owner_safe = False
|
||||
native = invoke(adapter.add, staged, root/'add')
|
||||
@@ -413,6 +413,7 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
return finish('RECOVERY_REQUIRED', '原生新增后名称、分区或工程关联未确认一致。\n'+native.detail, owner_safe)
|
||||
uploaded = True
|
||||
checkpoint('uploaded')
|
||||
native_stack.close()
|
||||
if not request.auto_verify:
|
||||
return finish('UPLOADED', '上传已确认;本批次未启用自动校验。\n'+native.detail)
|
||||
# Adapter catalog cannot return until its exact native owner exits.
|
||||
@@ -442,29 +443,29 @@ def run_management(request, *, adapter=None, verifier=None, idle_check=None, pro
|
||||
status = 'BLOCKED'
|
||||
return finish(status, str(error), owner_safe)
|
||||
finally:
|
||||
if mutex is not None:
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
uninitialize_com(initialized)
|
||||
|
||||
|
||||
def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, **kwargs):
|
||||
def run_management_batch(requests, *, stop=None, progress=None, item_executor=None, context=None, **kwargs):
|
||||
from threading import Event
|
||||
from .write_admission import WriteAdmission
|
||||
from .operation_queue import QueueCancelled, TaskHandoff, request_library_keys
|
||||
if context is not None:
|
||||
stop = stop if stop is not None else context.stop
|
||||
progress = context.guard_callback(progress) if progress else None
|
||||
if stop is None:
|
||||
stop = Event()
|
||||
kwargs.setdefault('write_admission',WriteAdmission(prompt=kwargs.get('version_prompt'),stop=stop))
|
||||
results = []
|
||||
retained_libraries = set()
|
||||
cancelled = False
|
||||
for request in requests:
|
||||
if stop is not None and stop.is_set():
|
||||
break
|
||||
key = str(request.library).casefold()
|
||||
if key in retained_libraries:
|
||||
keys = request_library_keys(request, context)
|
||||
stopped = cancelled or stop.is_set() or context is not None and context.stop.is_set()
|
||||
if stopped or keys.intersection(retained_libraries):
|
||||
result = ManagementResult(request.operation, request.library, request.name, request.partition,
|
||||
request.source, request.auto_verify, 'complete', 'SKIPPED', True,
|
||||
'同库原生会话尚未释放,未启动新的写入')
|
||||
'队列已停止,未操作此 CBB' if stopped else '同库原生会话尚未释放,未启动新的写入')
|
||||
results.append(result)
|
||||
if progress:
|
||||
progress({'stage': result.status, 'block': result.name, 'detail': result.detail})
|
||||
@@ -473,21 +474,21 @@ def run_management_batch(requests, *, stop=None, progress=None, item_executor=No
|
||||
result = run_management(request, progress=progress, stop=stop, **kwargs)
|
||||
if not result.safe_to_continue:
|
||||
from .operation_lifecycle import recheck_failed_release
|
||||
result = recheck_failed_release(request.library, request.name, result)
|
||||
result = recheck_failed_release(request.library, request.name, result, stop=stop)
|
||||
return result
|
||||
from .operation_queue import QueueCancelled
|
||||
try:
|
||||
result = item_executor(request, action) if item_executor is not None else action()
|
||||
except QueueCancelled as error:
|
||||
results.append(ManagementResult(request.operation, request.library, request.name, request.partition,
|
||||
request.source, request.auto_verify, 'complete', 'CANCELLED', True, str(error)))
|
||||
break
|
||||
result = ManagementResult(request.operation, request.library, request.name, request.partition,
|
||||
request.source, request.auto_verify, 'complete',
|
||||
'TIMEOUT' if isinstance(error, TaskHandoff) and not error.cancelled else 'CANCELLED',
|
||||
not isinstance(error, TaskHandoff), str(error))
|
||||
results.append(result)
|
||||
if progress:
|
||||
progress({'stage': result.status, 'block': result.name, 'detail': result.detail,
|
||||
'evidence_path': str(result.evidence_path or '')})
|
||||
if not result.safe_to_continue:
|
||||
retained_libraries.add(key)
|
||||
retained_libraries.update(keys)
|
||||
if result.status == 'CANCELLED':
|
||||
break
|
||||
cancelled = True
|
||||
return results
|
||||
@@ -1285,6 +1285,8 @@ def management_worker_main(request_path):
|
||||
except (FileNotFoundError, PermissionError):
|
||||
result['safe_to_continue'] = True
|
||||
raise
|
||||
from .desktop_worker import register_native_writer
|
||||
register_native_writer(process)
|
||||
update_worker_state(job, 'NATIVE_CALL', identities=[('native', process.pid)], continuation=True)
|
||||
try:
|
||||
_write_json(root/'native-process.json', {'pid': process.pid, 'command': command})
|
||||
|
||||
@@ -264,15 +264,34 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr
|
||||
job_path = root/'repair-request.json'
|
||||
cancel = root/'cancel-repair.signal'
|
||||
desktop = 'CbbRepair_'+uuid4().hex[:12]
|
||||
job_path.write_text(json.dumps({'project': str(target.project), 'database': str(target.database),
|
||||
from .operation_queue import current_context
|
||||
context = current_context()
|
||||
library = (maintenance_context or {}).get('library') or (context.library if context else str(target.project))
|
||||
job = {'project': str(target.project), 'database': str(target.database),
|
||||
'root': str(root), 'mentor_bin': str(mentor_bin or ''), 'timeout': timeout_seconds,
|
||||
'cancel_signal': str(cancel), 'supervisor_pid': os.getpid(), 'desktop': desktop,
|
||||
'maintenance_context': maintenance_context, 'inspection_only': inspection_only},
|
||||
ensure_ascii=False), encoding='utf-8')
|
||||
'maintenance_context': maintenance_context, 'inspection_only': inspection_only,
|
||||
'library': library, 'operation_id': root.name+'-'+uuid4().hex[:8],
|
||||
'operation': 'inspect' if inspection_only else 'repair',
|
||||
'block': (maintenance_context or {}).get('block'),
|
||||
'session_record_path': str(root/'session.json'), 'start_gate': str(root/'start-repair.signal')}
|
||||
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
|
||||
command = ([sys.executable] if getattr(sys, 'frozen', False) else
|
||||
[sys.executable, '-m', 'cbb_refresh_verify']) + ['--repair-worker', str(job_path)]
|
||||
from .operation_queue import native_channel, task_stop
|
||||
with native_channel(inspection_only=inspection_only is True):
|
||||
return _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, task_stop(stop), job)
|
||||
|
||||
|
||||
def _supervise_repair(command, desktop, cancel, root, timeout_seconds, progress, stop, job):
|
||||
from .desktop_worker import DesktopWorker
|
||||
worker = DesktopWorker(command, desktop)
|
||||
worker.bind_cancellation(cancel)
|
||||
if hasattr(worker, 'bind_session'):
|
||||
worker.bind_session(job)
|
||||
from .operation_queue import check_active
|
||||
check_active()
|
||||
Path(job['start_gate']).touch()
|
||||
# Worker admission includes filesystem/ownership checks before the native
|
||||
# timeout begins. Bound that preparation separately; never extend on polls.
|
||||
deadline = time.monotonic()+60+max(1, float(timeout_seconds))+5
|
||||
@@ -307,4 +326,9 @@ def run_native_repair(target, root, *, mentor_bin='', timeout_seconds=300, progr
|
||||
result['status'] = 'UNCONFIRMED'
|
||||
return {**result, 'evidence_path': str(root)}
|
||||
finally:
|
||||
if worker.poll() is None and getattr(worker, 'session_record_path', None):
|
||||
from .session_record import handoff_session
|
||||
from .operation_queue import hold_native_handoff
|
||||
handoff_session(worker.session_record_path, evidence_root=root, process=worker)
|
||||
hold_native_handoff(worker, worker.session_record_path)
|
||||
worker.close()
|
||||
@@ -335,6 +335,8 @@ def repair_worker_main(job_path):
|
||||
if Path(job['cancel_signal']).exists():
|
||||
raise ValueError('REPAIR_CANCELLED_BEFORE_LAUNCH')
|
||||
process = subprocess.Popen([str(executable), '-p', str(project)], cwd=str(project.parent))
|
||||
from .desktop_worker import register_native_writer
|
||||
register_native_writer(process)
|
||||
result['native_started'] = True
|
||||
result['native_pid'] = process.pid
|
||||
emit(stage='native_started', pid=process.pid, project=str(project))
|
||||
|
||||
@@ -2,10 +2,19 @@
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
def recheck_failed_release(library, block, result):
|
||||
def recheck_failed_release(library, block, result, *, stop=None, timeout_seconds=10):
|
||||
"""Refresh resource facts after a late exit; keep the original business failure."""
|
||||
if result.safe_to_continue or not result.evidence_path:
|
||||
return result
|
||||
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
|
||||
if current_context() is None:
|
||||
context = BatchTaskContext(library=str(library), targets=(block,))
|
||||
try:
|
||||
return context.run(lambda: recheck_failed_release(library, block, result, stop=stop),
|
||||
stop=stop, timeout_seconds=timeout_seconds)
|
||||
except TaskHandoff:
|
||||
return result
|
||||
stop = task_stop(stop)
|
||||
from pathlib import Path
|
||||
from dataclasses import replace
|
||||
import json
|
||||
@@ -37,9 +46,16 @@ def recheck_failed_release(library, block, result):
|
||||
bound.append(candidate)
|
||||
if not bound:
|
||||
return result # A generic timeout without a bound owner is not an exit fact.
|
||||
proof = release_retained_library(library, evidence.parent/'late-release', block_name=block)
|
||||
if proof.get('status') != 'RELEASE_CONFIRMED' or Path(proof.get('library', '')).resolve() != Path(library).resolve():
|
||||
from .operation_queue import check_active
|
||||
check_active()
|
||||
proof = release_retained_library(library, evidence.parent/'late-release', block_name=block, stop=stop)
|
||||
scope = proof.get('resource_scope')
|
||||
if (proof.get('status') != 'RELEASE_CONFIRMED' or proof.get('lease_state') != 'RELEASED'
|
||||
or not isinstance(scope, dict) or scope.get('scope_state') != 'OBSERVED'
|
||||
or proof.get('blocked_resources') != []
|
||||
or Path(proof.get('library', '')).resolve() != Path(library).resolve()):
|
||||
return result
|
||||
check_active()
|
||||
evidence.with_name('late-release-result.json').write_text(json.dumps(dict(
|
||||
business_status=result.status, safe_to_continue=True, release_proof=proof),
|
||||
ensure_ascii=False, default=str, indent=2), 'utf-8')
|
||||
|
||||
@@ -1,36 +1,320 @@
|
||||
"""Cooperative FIFO admission for this application's native operations."""
|
||||
from threading import Condition
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
import ntpath
|
||||
from threading import Condition, Event, RLock, Thread, local
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
class QueueCancelled(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class TaskHandoff(QueueCancelled):
|
||||
"""The call is still observable; this is never a release receipt."""
|
||||
def __init__(self, context, *, cancelled=False, result=None):
|
||||
self.operation_id = context.operation_id
|
||||
self.library = context.library
|
||||
self.cancelled = cancelled
|
||||
self.scope = context.scope
|
||||
self.result = result
|
||||
super().__init__('已交给后台观察;当前资源尚未确认释放。')
|
||||
|
||||
|
||||
class _StopSignal:
|
||||
def __init__(self):
|
||||
self._event = Event()
|
||||
|
||||
def set(self):
|
||||
self._event.set()
|
||||
|
||||
def is_set(self):
|
||||
return self._event.is_set()
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return self._event.wait(timeout)
|
||||
|
||||
|
||||
_local = local()
|
||||
_tasks_guard = RLock()
|
||||
_calls = {}
|
||||
|
||||
|
||||
def _path_key(path):
|
||||
# Identity was bound by the caller; queue predicates must never resolve a share.
|
||||
return ntpath.normcase(ntpath.normpath(str(path)))
|
||||
|
||||
|
||||
def request_library_keys(request, context=None):
|
||||
library = str(getattr(request, 'lmc_file', getattr(request, 'library', '__injected_verifier_scope__')))
|
||||
keys = {_path_key(library)}
|
||||
drive, remote = getattr(request, 'mapped_drive', ''), getattr(request, 'mapped_remote_root', '')
|
||||
if drive and remote and ntpath.splitdrive(library)[0].casefold() == drive.rstrip('\\/').casefold():
|
||||
keys.add(_path_key(ntpath.join(remote, ntpath.splitdrive(library)[1].lstrip('\\/'))))
|
||||
if context is not None:
|
||||
aliases = {_path_key(v) for v in (context.library, *context.aliases)}
|
||||
if keys.intersection(aliases):
|
||||
keys.update(aliases)
|
||||
return keys
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class _Call:
|
||||
context: object
|
||||
external_stop: object = None
|
||||
done: Event = field(default_factory=Event)
|
||||
abandoned: bool = False
|
||||
native_depth: int = 0
|
||||
native_handoff: str = ''
|
||||
phase_started: float = field(default_factory=time.monotonic)
|
||||
result: object = None
|
||||
last_result: object = None
|
||||
error: BaseException | None = None
|
||||
|
||||
def is_set(self):
|
||||
return (self.abandoned or self.context.stop.is_set()
|
||||
or self.external_stop is not None and self.external_stop.is_set())
|
||||
|
||||
|
||||
def current_context():
|
||||
call = getattr(_local, 'call', None)
|
||||
return call.context if call is not None else None
|
||||
|
||||
|
||||
def task_stop(fallback=None):
|
||||
return getattr(_local, 'call', None) or fallback
|
||||
|
||||
|
||||
def check_active():
|
||||
call = getattr(_local, 'call', None)
|
||||
if call is not None and call.is_set():
|
||||
raise QueueCancelled('任务已取消,未提交新的原生操作。')
|
||||
|
||||
|
||||
def remember_result(result):
|
||||
call = getattr(_local, 'call', None)
|
||||
if call is not None:
|
||||
with _tasks_guard:
|
||||
call.last_result = result
|
||||
return result
|
||||
|
||||
|
||||
def hold_native_handoff(process, session_record_path):
|
||||
"""Keep this thread's mutex while a recorded native client is unreturned.
|
||||
|
||||
The caller registers the observer first. Only UI/scheduling is handed off;
|
||||
this thread and its native channel remain alive until that exact handle exits.
|
||||
"""
|
||||
call = getattr(_local, 'call', None)
|
||||
if call is not None:
|
||||
with _tasks_guard:
|
||||
if not call.native_depth:
|
||||
raise RuntimeError('Native handoff requires an owned native phase')
|
||||
call.native_handoff = str(session_record_path)
|
||||
while process.poll() is None:
|
||||
Event().wait(.1)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def native_phase():
|
||||
"""Never abandon a transaction between native writes (including Remove/Add)."""
|
||||
call = getattr(_local, 'call', None)
|
||||
with _tasks_guard:
|
||||
check_active()
|
||||
if call is not None:
|
||||
call.native_depth += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if call is not None:
|
||||
with _tasks_guard:
|
||||
call.native_depth -= 1
|
||||
call.phase_started = time.monotonic()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def native_channel(*, inspection_only=False):
|
||||
"""Shared by Verify, Repair and backup native clients; reentrant per thread."""
|
||||
import win32con, win32event
|
||||
with native_phase():
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
try:
|
||||
check_active()
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
raise RuntimeError('NATIVE_CHANNEL_BUSY: 另一个原生操作仍在运行。')
|
||||
from .desktop_worker import assert_native_writer_idle
|
||||
if inspection_only is not True:
|
||||
assert_native_writer_idle()
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchTaskContext:
|
||||
library: str
|
||||
aliases: tuple[str, ...] = ()
|
||||
targets: tuple[str, ...] = ()
|
||||
resources: tuple[str, ...] = ()
|
||||
operation_id: str = field(default_factory=lambda: uuid4().hex)
|
||||
stop: _StopSignal = field(default_factory=_StopSignal, repr=False, compare=False)
|
||||
_closed: Event = field(default_factory=Event, repr=False, compare=False)
|
||||
|
||||
def __post_init__(self):
|
||||
object.__setattr__(self, 'library', str(self.library))
|
||||
for name in ('aliases', 'targets', 'resources'):
|
||||
object.__setattr__(self, name, tuple(str(v) for v in getattr(self, name)))
|
||||
|
||||
@property
|
||||
def scope(self):
|
||||
return dict(scope_state='DECLARED', library=self.library, aliases=list(self.aliases),
|
||||
block_names=list(self.targets), resources=list(self.resources),
|
||||
operation_ids=[self.operation_id])
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self.snapshot()['state']
|
||||
|
||||
def snapshot(self):
|
||||
with _tasks_guard:
|
||||
calls = [call for call in _calls.values() if call.context is self]
|
||||
pending = sum(not call.done.is_set() for call in calls)
|
||||
return dict(operation_id=self.operation_id, resource_scope=self.scope,
|
||||
state='CLOSED' if self._closed.is_set() else
|
||||
'CANCELLED' if self.stop.is_set() else
|
||||
'HANDED_OFF' if any(c.abandoned for c in calls) else
|
||||
'RUNNING' if pending else 'IDLE', pending=pending,
|
||||
pending_native=any(c.native_depth for c in calls))
|
||||
|
||||
def cancel(self):
|
||||
self.stop.set()
|
||||
|
||||
def check_active(self):
|
||||
if self.stop.is_set() or self._closed.is_set():
|
||||
raise QueueCancelled('任务已取消,未提交新的原生操作。')
|
||||
check_active()
|
||||
|
||||
def guard_callback(self, callback, *, current=None):
|
||||
call = getattr(_local, 'call', None)
|
||||
def guarded(*args, **kwargs):
|
||||
# Cancellation still gets its own terminal result; closing or an
|
||||
# abandoned invocation cannot update a newer task/library.
|
||||
if (not self._closed.is_set() and not (call and call.abandoned)
|
||||
and (current is None or current() is self)):
|
||||
return callback(*args, **kwargs)
|
||||
return guarded
|
||||
|
||||
def close(self, *, timeout_seconds=30):
|
||||
self._closed.set()
|
||||
self.cancel()
|
||||
deadline = time.monotonic() + min(30, max(0, timeout_seconds))
|
||||
with _tasks_guard:
|
||||
calls = [c for c in _calls.values() if c.context is self]
|
||||
for call in calls:
|
||||
call.done.wait(max(0, deadline-time.monotonic()))
|
||||
with _tasks_guard:
|
||||
for call in calls:
|
||||
if not call.done.is_set() and not call.native_depth:
|
||||
call.abandoned = True
|
||||
return self.snapshot()
|
||||
|
||||
def run(self, action, *, stop=None, timeout_seconds=10):
|
||||
self.check_active()
|
||||
if current_context() is self:
|
||||
return action() # Preserve nested lease and mutex thread identity.
|
||||
aliases = {_path_key(v) for v in (self.library, *self.aliases)}
|
||||
call = _Call(self, stop)
|
||||
with _tasks_guard:
|
||||
if any(c.abandoned and aliases.intersection(
|
||||
_path_key(v) for v in (c.context.library, *c.context.aliases))
|
||||
for c in _calls.values()):
|
||||
raise TaskHandoff(self)
|
||||
_calls[id(call)] = call
|
||||
def execute():
|
||||
_local.call = call
|
||||
try:
|
||||
check_active()
|
||||
call.result = action()
|
||||
except BaseException as error:
|
||||
call.error = error
|
||||
finally:
|
||||
with _tasks_guard:
|
||||
call.done.set()
|
||||
_calls.pop(id(call), None)
|
||||
del _local.call
|
||||
Thread(target=execute, name='cbb-task-'+self.operation_id[:12], daemon=True).start()
|
||||
while not call.done.wait(.02):
|
||||
with _tasks_guard:
|
||||
if call.native_handoff:
|
||||
cancelled = call.is_set()
|
||||
call.abandoned = True
|
||||
raise TaskHandoff(self, cancelled=cancelled, result=call.last_result)
|
||||
elapsed = time.monotonic()-call.phase_started
|
||||
if not call.native_depth and (call.abandoned or elapsed >= timeout_seconds):
|
||||
cancelled = call.is_set()
|
||||
call.abandoned = True
|
||||
raise TaskHandoff(self, cancelled=cancelled, result=call.last_result)
|
||||
if call.error is not None:
|
||||
raise call.error
|
||||
return call.result
|
||||
|
||||
|
||||
class NativeQueue:
|
||||
def __init__(self):
|
||||
self.condition = Condition()
|
||||
self.pending = []
|
||||
self.active = False
|
||||
self.closed = False
|
||||
self.contexts = {}
|
||||
|
||||
def wake(self):
|
||||
with self.condition: self.condition.notify_all()
|
||||
|
||||
def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None):
|
||||
ticket = object()
|
||||
def close(self, *, timeout_seconds=30):
|
||||
deadline = time.monotonic()+min(30, max(0, timeout_seconds))
|
||||
with self.condition:
|
||||
self.closed = True
|
||||
contexts = tuple(self.contexts.values())
|
||||
for context in contexts:
|
||||
context.cancel()
|
||||
self.condition.notify_all()
|
||||
return tuple(context.close(timeout_seconds=max(0, deadline-time.monotonic()))
|
||||
for context in contexts)
|
||||
|
||||
def run(self, action, *, stop=None, waiting=None, ready=lambda: True, wait_state=None,
|
||||
context=None, timeout_seconds=10):
|
||||
ticket = [object(), False]
|
||||
acquired, notified = False, False
|
||||
previous_state = None
|
||||
with self.condition: self.pending.append((ticket, ready))
|
||||
with self.condition:
|
||||
if self.closed:
|
||||
raise QueueCancelled('队列已关闭,未启动新的操作。')
|
||||
if context is not None:
|
||||
self.contexts[id(context)] = context
|
||||
self.pending.append(ticket)
|
||||
try:
|
||||
while not acquired:
|
||||
available = ready() # Call only this ticket's predicate, outside the queue lock.
|
||||
with self.condition:
|
||||
if stop is not None and stop.is_set():
|
||||
if (self.closed or stop is not None and stop.is_set()
|
||||
or context is not None and context.stop.is_set()):
|
||||
raise QueueCancelled('已取消排队,未启动原生操作。')
|
||||
first = next((token for token, available in self.pending if available()), None)
|
||||
ticket[1] = bool(available)
|
||||
first = next((entry for entry in self.pending if entry[1]), None)
|
||||
if not self.active and first is ticket:
|
||||
self.pending = [entry for entry in self.pending if entry[0] is not ticket]
|
||||
self.active = acquired = True
|
||||
else:
|
||||
self.condition.wait(.1)
|
||||
if acquired and not ready():
|
||||
with self.condition:
|
||||
ticket[1] = self.active = acquired = False
|
||||
self.condition.notify_all()
|
||||
if not acquired and waiting is not None:
|
||||
try:
|
||||
current_state = wait_state() if wait_state is not None else None
|
||||
@@ -41,9 +325,9 @@ class NativeQueue:
|
||||
pass
|
||||
if stop is not None and stop.is_set():
|
||||
raise QueueCancelled('已取消排队,未启动原生操作。')
|
||||
return action()
|
||||
return context.run(action, stop=stop, timeout_seconds=timeout_seconds) if context else action()
|
||||
finally:
|
||||
with self.condition:
|
||||
self.pending = [entry for entry in self.pending if entry[0] is not ticket]
|
||||
self.pending = [entry for entry in self.pending if entry is not ticket]
|
||||
if acquired: self.active = False
|
||||
self.condition.notify_all()
|
||||
@@ -130,7 +130,7 @@ def register_worker_session(worker, job):
|
||||
path = Path(job['session_record_path'])
|
||||
from .icdb_ownership import session_resource_scope
|
||||
write_session_record(path, {k: job.get(k) for k in (
|
||||
'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal')}
|
||||
'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal', 'worker_lease_handle')}
|
||||
| {'processes': [{'role': 'worker', **process_identity(worker.pid)}],
|
||||
'lease_state': 'ACTIVE', 'resource_scope': session_resource_scope(job),
|
||||
'blocked_resources': [], 'data_recovery_required': None})
|
||||
|
||||
@@ -545,14 +545,57 @@ def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline, l
|
||||
|
||||
|
||||
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
# Cover contents checking, native work and final state reads with one
|
||||
# session-wide mutex. The worker-stage lock below is reentrant on Windows.
|
||||
import win32event, win32con
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None,
|
||||
admission_timeout_seconds=10):
|
||||
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
|
||||
options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner,
|
||||
port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory,
|
||||
version_prompt=version_prompt)
|
||||
bound = current_context()
|
||||
if bound is None:
|
||||
# Management may call Verify while holding a same-thread lease. Do not
|
||||
# transfer that transaction to a different thread.
|
||||
import threading
|
||||
from .library_lock import _guard, _owners
|
||||
with _guard:
|
||||
nested = any(v['thread'] == threading.get_ident() and v['depth'] > 0
|
||||
for v in _owners.values())
|
||||
if not nested:
|
||||
from .operation_queue import request_library_keys
|
||||
bound = context or BatchTaskContext(library=str(request.lmc_file),
|
||||
aliases=tuple(sorted(request_library_keys(request))), targets=(request.block_name,))
|
||||
try:
|
||||
return bound.run(lambda: run_verify(request, stop=stop, **options),
|
||||
stop=stop, timeout_seconds=admission_timeout_seconds)
|
||||
except TaskHandoff as error:
|
||||
result = _handoff_verify_result(error)
|
||||
_notify_progress(bound.guard_callback(progress) if progress else None,
|
||||
dict(stage='task_handoff', operation_id=bound.operation_id,
|
||||
library=bound.library, status=result.status, resource_scope=error.scope))
|
||||
return result
|
||||
if bound is not None and progress is not None:
|
||||
options['progress'] = bound.guard_callback(progress)
|
||||
return _run_verify_task(request, stop=task_stop(stop), **options)
|
||||
|
||||
|
||||
def _handoff_verify_result(error):
|
||||
known = error.result if isinstance(error.result, VerifyResult) else None
|
||||
result = known or VerifyResult('CANCELLED' if error.cancelled else 'TIMEOUT')
|
||||
return replace(result, status='FAIL' if result.status == 'PASS' else result.status,
|
||||
stderr=result.stderr or str(error), safe_to_continue=False,
|
||||
error_stage=result.error_stage or 'admission_or_finalization',
|
||||
error_category=result.error_category or 'TASK_HANDOFF',
|
||||
release_proof=dict(lease_state='PENDING', resource_scope=error.scope,
|
||||
blocked_resources=error.scope.get('resources', [])))
|
||||
|
||||
|
||||
def _run_verify_task(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
# Admission/finalization keep only the library lease. Native launch paths
|
||||
# acquire the host channel after slow metadata has returned.
|
||||
from .operation_queue import check_active
|
||||
from .com_runtime import initialize_com, uninitialize_com
|
||||
initialized = initialize_com()
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
library_lock = None
|
||||
previous_release = None
|
||||
release_confirmed = True
|
||||
@@ -563,11 +606,7 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
from .library_lock import _supervisor_alive
|
||||
return bool(remaining_clients) and not any(_supervisor_alive(pid) for pid in remaining_clients)
|
||||
try:
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
if not acquired:
|
||||
return VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。',
|
||||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY')
|
||||
check_active()
|
||||
if request.maintenance:
|
||||
from .maintenance_entry import run_icdb_check
|
||||
return run_icdb_check(request, progress=progress, stop=stop)
|
||||
@@ -576,10 +615,13 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
if Path(request.lmc_file).is_file():
|
||||
try:
|
||||
previous_release = recheck_previous_release(request.lmc_file,
|
||||
Path(request.evidence_dir)/'previous-release', block_name=request.block_name, progress=progress)
|
||||
Path(request.evidence_dir)/'previous-release', block_name=request.block_name,
|
||||
progress=progress, stop=stop)
|
||||
check_active()
|
||||
library_lock = LibraryLock(request.lmc_file, checkpoint=False,
|
||||
safe_to_release=clients_released)
|
||||
library_lock.__enter__()
|
||||
check_active()
|
||||
except LibraryBusy as error:
|
||||
return VerifyResult('FAIL', stderr=str(error), error_category='LIBRARY_IN_USE')
|
||||
except (OSError, ValueError, RuntimeError, pywintypes.error) as error:
|
||||
@@ -691,14 +733,12 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
if library_lock is not None:
|
||||
library_lock.__exit__(None, None, None)
|
||||
finally:
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
uninitialize_com(initialized)
|
||||
|
||||
|
||||
def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
port_check=None, dialog_supervisor_factory=None, version_prompt=None, stop=None):
|
||||
from .operation_queue import remember_result
|
||||
timing = RunTimings()
|
||||
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
|
||||
process_runner=process_runner, port_check=port_check,
|
||||
@@ -722,6 +762,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
|
||||
result = replace(result, status='FAIL', stderr='原生会话或自动化客户端尚未退出。')
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
remember_result(result)
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP', 'CANCELLED'} and result.evidence_path:
|
||||
job_path = result.evidence_path.with_name('request.json')
|
||||
if job_path.is_file():
|
||||
@@ -764,6 +805,7 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
|
||||
if result.status not in {"PASS", "SKIPPED"} else ""),
|
||||
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
|
||||
if result.status not in {"PASS", "SKIPPED"} else ""))
|
||||
remember_result(result)
|
||||
if result.evidence_path:
|
||||
_write_optional_evidence(result.evidence_path, {'block_name': request.block_name, **asdict(result)})
|
||||
_write_optional_evidence(result.evidence_path.with_name('performance.json'), {
|
||||
@@ -819,7 +861,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
except Exception:
|
||||
# Cleanup must never hide or change the verification result.
|
||||
pass
|
||||
return replace(result, evidence_path=result_path)
|
||||
from .operation_queue import remember_result
|
||||
return remember_result(replace(result, evidence_path=result_path))
|
||||
|
||||
if stop is not None and stop.is_set():
|
||||
return finish(VerifyResult('CANCELLED', stderr='已取消,未启动校验。'))
|
||||
@@ -843,7 +886,9 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
'detail': ' '.join(command)})
|
||||
timing.begin('content_check')
|
||||
try:
|
||||
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
|
||||
from .operation_queue import native_channel
|
||||
with native_channel():
|
||||
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
|
||||
finally:
|
||||
timing.end('content_check')
|
||||
combined = '\n'.join(part for part in (stdout, stderr) if part)
|
||||
@@ -915,17 +960,15 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
from .desktop_worker import DesktopWorker
|
||||
from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend
|
||||
from .dialog_models import DialogPhase
|
||||
import win32event, win32con
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
lock_status = win32event.WaitForSingleObject(mutex, 0)
|
||||
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
|
||||
mutex.Close()
|
||||
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行;本次未启动。',
|
||||
safe_to_continue=True, error_category='NATIVE_CHANNEL_BUSY'))
|
||||
from .operation_queue import check_active, native_channel
|
||||
check_active()
|
||||
worker_launch_attempted = False
|
||||
native = native_channel()
|
||||
phase_entered = False
|
||||
try:
|
||||
timing.begin('worker_setup')
|
||||
with (root/'worker.log').open('w', encoding='utf-8') as log:
|
||||
check_active()
|
||||
worker_launch_attempted = True
|
||||
if request.interactive_input:
|
||||
process = DesktopWorker(
|
||||
@@ -965,6 +1008,13 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
)
|
||||
if stop is not None and stop.is_set():
|
||||
raise InterruptedError('VERIFY_CANCELLED: 启动已取消,未启动 PCB')
|
||||
check_active()
|
||||
try:
|
||||
native.__enter__()
|
||||
phase_entered = True
|
||||
except RuntimeError as error:
|
||||
return finish(VerifyResult('BLOCKED', stderr=str(error), safe_to_continue=False,
|
||||
error_category='NATIVE_CHANNEL_BUSY'))
|
||||
Path(job['start_gate']).touch()
|
||||
while process.poll() is None:
|
||||
if stop is not None and stop.is_set():
|
||||
@@ -1188,6 +1238,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
{'stage': 'session_observation', 'block': request.block_name, **state}))
|
||||
_notify_progress(progress, {'stage': 'session_handoff', 'block': request.block_name,
|
||||
'detail': '已转后台观察,可查看状态'})
|
||||
if getattr(process, 'native_write_pending', lambda: None)() is not False:
|
||||
from .operation_queue import hold_native_handoff
|
||||
hold_native_handoff(process, process.session_record_path)
|
||||
worker_exited = process.poll() is not None
|
||||
except Exception as error:
|
||||
close_errors.append(str(error))
|
||||
if 'process' in locals():
|
||||
@@ -1279,8 +1333,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
if phase_entered:
|
||||
native.__exit__(None, None, None)
|
||||
|
||||
if result.status == 'PASS' and not request.diagnostic_only:
|
||||
timing.begin('library_flag')
|
||||
@@ -1290,9 +1344,10 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
if progress:
|
||||
_notify_progress(progress, {'stage': 'library_flag_probe', 'block': request.block_name, **record})
|
||||
try:
|
||||
probe = read_library_flag_with_retry(
|
||||
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
|
||||
request.block_name, on_attempt=record_attempt)
|
||||
with native_channel():
|
||||
probe = read_library_flag_with_retry(
|
||||
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
|
||||
request.block_name, on_attempt=record_attempt)
|
||||
result = apply_post_verify_status(result, probe)
|
||||
result = replace(result, bridge_attempts=tuple(bridge_records))
|
||||
_write_optional_evidence(root/'bridge-status.json', bridge_records)
|
||||
@@ -1304,20 +1359,27 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
return finish(result)
|
||||
|
||||
|
||||
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None):
|
||||
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None, item_executor=None,
|
||||
context=None):
|
||||
"""Run serially and stop when database/session safety is unconfirmed."""
|
||||
if max_workers != 1:
|
||||
raise ValueError('Only serial Verify is supported for the shared Library Manager session')
|
||||
if context is not None:
|
||||
stop = stop if stop is not None else context.stop
|
||||
progress = context.guard_callback(progress) if progress is not None else None
|
||||
if verifier is None:
|
||||
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available, stop=stop) # noqa: E731
|
||||
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available,
|
||||
stop=stop, context=context) # noqa: E731
|
||||
results = []
|
||||
queue_stop = False
|
||||
retained_libraries = set()
|
||||
for request in requests:
|
||||
library_key = str(getattr(request, 'lmc_file', '__injected_verifier_scope__')).casefold()
|
||||
if queue_stop or (stop is not None and stop.is_set()):
|
||||
from .operation_queue import request_library_keys
|
||||
library_keys = request_library_keys(request, context)
|
||||
if (queue_stop or stop is not None and stop.is_set()
|
||||
or context is not None and context.stop.is_set()):
|
||||
result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB')
|
||||
elif library_key in retained_libraries:
|
||||
elif library_keys.intersection(retained_libraries):
|
||||
result = VerifyResult('SKIPPED', stderr='同库原生会话尚未释放,未启动新的写入')
|
||||
else:
|
||||
try:
|
||||
@@ -1325,18 +1387,19 @@ def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=Non
|
||||
result = verifier(request)
|
||||
if not result.safe_to_continue:
|
||||
from .operation_lifecycle import recheck_failed_release
|
||||
result = recheck_failed_release(request.lmc_file, request.block_name, result)
|
||||
result = recheck_failed_release(request.lmc_file, request.block_name, result, stop=stop)
|
||||
return result
|
||||
result = item_executor(request, execute) if item_executor is not None else execute()
|
||||
except Exception as error:
|
||||
from .operation_queue import QueueCancelled
|
||||
from .operation_queue import QueueCancelled, TaskHandoff
|
||||
# An unexpected exception cannot establish database safety.
|
||||
result = VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL',
|
||||
stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled))
|
||||
result = (_handoff_verify_result(error) if isinstance(error, TaskHandoff) else
|
||||
VerifyResult('CANCELLED' if isinstance(error, QueueCancelled) else 'FAIL',
|
||||
stderr=str(error), safe_to_continue=isinstance(error, QueueCancelled)))
|
||||
results.append(result)
|
||||
# Never start another writer after an unconfirmed native shutdown.
|
||||
if not result.safe_to_continue:
|
||||
retained_libraries.add(library_key)
|
||||
retained_libraries.update(library_keys)
|
||||
queue_stop = queue_stop or result.status in {'QUEUE_STOP', 'CANCELLED'}
|
||||
if queue_stop and stop is not None:
|
||||
stop.set()
|
||||
|
||||
@@ -71,7 +71,7 @@ def test_verify_failure_recheck_finishes_before_yielding_native_queue(qtbot,tmp_
|
||||
try:return action()
|
||||
finally:active.pop()
|
||||
return run
|
||||
def recheck(library,block,result):
|
||||
def recheck(library,block,result,**kwargs):
|
||||
seen.append(bool(active))
|
||||
return replace(result,safe_to_continue=True)
|
||||
monkeypatch.setattr(window,'_queued_action',queued)
|
||||
@@ -92,7 +92,7 @@ def test_management_failure_recheck_finishes_before_yielding_native_queue(tmp_pa
|
||||
active.append(True)
|
||||
try:return action()
|
||||
finally:active.pop()
|
||||
def recheck(library,block,result):
|
||||
def recheck(library,block,result,**kwargs):
|
||||
seen.append(bool(active))
|
||||
return replace(result,safe_to_continue=True)
|
||||
monkeypatch.setattr(operation_lifecycle,'recheck_failed_release',recheck)
|
||||
|
||||
@@ -0,0 +1,724 @@
|
||||
"""Cancellation must release scheduling, never invent native release."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import win32con
|
||||
import win32event
|
||||
|
||||
from cbb_refresh_verify import library_lock as locks
|
||||
from test_management import setup as management_setup
|
||||
|
||||
|
||||
@pytest.mark.parametrize('stage', ['read', 'archive'])
|
||||
def test_stalled_checkpoint_does_not_own_native_channel_or_commit_after_stop(tmp_path, monkeypatch, stage):
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease:
|
||||
pass
|
||||
marker = locks.blocker_path(library)
|
||||
original = marker.read_bytes()
|
||||
entered, resume, stop = Event(), Event(), Event()
|
||||
read, write = Path.read_bytes, Path.write_bytes
|
||||
def read_bytes(path):
|
||||
if stage == 'read' and path == marker and not resume.is_set():
|
||||
entered.set()
|
||||
assert resume.wait(5)
|
||||
return read(path)
|
||||
def write_bytes(path, data):
|
||||
if stage == 'archive' and path.name.startswith('blocker-before-'):
|
||||
entered.set()
|
||||
assert resume.wait(5)
|
||||
return write(path, data)
|
||||
monkeypatch.setattr(Path, 'read_bytes', read_bytes)
|
||||
monkeypatch.setattr(Path, 'write_bytes', write_bytes)
|
||||
monkeypatch.setattr('cbb_refresh_verify.owner_probe.query_owners', lambda *_: [])
|
||||
with ThreadPoolExecutor(1) as pool:
|
||||
task = pool.submit(locks.release_retained_library, library, tmp_path/'evidence', stop=stop)
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
try:
|
||||
assert entered.wait(3)
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) in (
|
||||
win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED)
|
||||
assert acquired, 'checkpoint I/O retained the host native channel'
|
||||
if stage == 'archive':
|
||||
import json
|
||||
receipt = next((tmp_path/'evidence').rglob('release.json'))
|
||||
assert json.loads(receipt.read_text())['lease_state'] != 'RELEASED'
|
||||
stop.set()
|
||||
resume.set()
|
||||
with pytest.raises(InterruptedError):
|
||||
task.result(5)
|
||||
assert read(marker) == original
|
||||
assert locks.has_retained_library(library)
|
||||
finally:
|
||||
stop.set()
|
||||
resume.set()
|
||||
if acquired:
|
||||
win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
try:
|
||||
task.result(5)
|
||||
except InterruptedError:
|
||||
pass
|
||||
with locks._guard:
|
||||
owner = locks._owners.pop(lease.key, None)
|
||||
if owner:
|
||||
owner['handle'].Close()
|
||||
|
||||
|
||||
def test_task_context_is_frozen_pure_and_has_an_independent_monotonic_stop(tmp_path, monkeypatch):
|
||||
from dataclasses import FrozenInstanceError
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext
|
||||
monkeypatch.setattr(Path, 'resolve', lambda *a, **k: pytest.fail('context performed filesystem admission'))
|
||||
first = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',), targets=('A',))
|
||||
second = BatchTaskContext(library=str(tmp_path/'B.lmc'))
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
first.library = second.library
|
||||
first.cancel()
|
||||
assert first.stop.is_set() and not second.stop.is_set()
|
||||
assert not hasattr(first.stop, 'clear')
|
||||
received = []
|
||||
first.guard_callback(received.append)('CANCELLED')
|
||||
assert received == ['CANCELLED']
|
||||
first.close(timeout_seconds=0)
|
||||
first.guard_callback(received.append)('late')
|
||||
assert received == ['CANCELLED']
|
||||
|
||||
|
||||
def test_bounded_queue_handoff_preserves_scope_and_fences_late_native_launch(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff, native_phase
|
||||
queue = NativeQueue()
|
||||
first = BatchTaskContext(library=str(tmp_path/'A.lmc'), targets=('A',))
|
||||
second = BatchTaskContext(library=str(tmp_path/'B.lmc'), targets=('B',))
|
||||
entered, resume, finished = Event(), Event(), Event()
|
||||
effects = []
|
||||
def stalled():
|
||||
try:
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
with native_phase():
|
||||
effects.append('late native write')
|
||||
finally:
|
||||
finished.set()
|
||||
try:
|
||||
with pytest.raises(TaskHandoff) as error:
|
||||
queue.run(stalled, context=first, timeout_seconds=.05)
|
||||
assert entered.is_set() and error.value.library == first.library
|
||||
assert first.snapshot()['pending'] == 1
|
||||
assert queue.run(lambda: 'B ran', context=second) == 'B ran'
|
||||
resume.set()
|
||||
assert finished.wait(2)
|
||||
assert effects == []
|
||||
finally:
|
||||
resume.set()
|
||||
assert finished.wait(2)
|
||||
|
||||
|
||||
def test_native_phase_is_never_yielded_at_metadata_deadline(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, native_phase
|
||||
queue = NativeQueue()
|
||||
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
|
||||
entered, resume, finished = Event(), Event(), Event()
|
||||
results = []
|
||||
def transaction():
|
||||
with native_phase():
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
return 'Remove and Add completed'
|
||||
def run():
|
||||
results.append(queue.run(transaction, context=context, timeout_seconds=.02))
|
||||
finished.set()
|
||||
thread = Thread(target=run)
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
assert not finished.wait(.1)
|
||||
state = context.close(timeout_seconds=.02)
|
||||
assert state['pending_native'] and queue.active
|
||||
resume.set()
|
||||
assert finished.wait(2)
|
||||
assert results == ['Remove and Add completed']
|
||||
finally:
|
||||
resume.set()
|
||||
thread.join(2)
|
||||
|
||||
|
||||
def test_stalled_ready_predicate_does_not_hold_queue_condition():
|
||||
from cbb_refresh_verify.operation_queue import NativeQueue, QueueCancelled
|
||||
queue = NativeQueue()
|
||||
entered, resume, stop, completed = Event(), Event(), Event(), Event()
|
||||
effects = []
|
||||
def slow_ready():
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
return True
|
||||
def first():
|
||||
try:
|
||||
queue.run(lambda: effects.append('A'), ready=slow_ready, stop=stop)
|
||||
except QueueCancelled:
|
||||
pass
|
||||
thread = Thread(target=first)
|
||||
thread.start()
|
||||
second = Thread(target=lambda: (queue.run(lambda: effects.append('B')), completed.set()))
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
second.start()
|
||||
assert completed.wait(.5), 'one ready predicate froze every other queue ticket'
|
||||
stop.set()
|
||||
resume.set()
|
||||
thread.join(2)
|
||||
assert effects == ['B']
|
||||
finally:
|
||||
stop.set()
|
||||
resume.set()
|
||||
thread.join(3)
|
||||
second.join(3)
|
||||
|
||||
|
||||
def test_verify_admission_handoff_unblocks_b_and_never_launches_after_read_returns(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
|
||||
from cbb_refresh_verify.verify_runner import VerifyRequest, run_verify
|
||||
import win32process
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
with locks.LibraryLock(library, block_name='A', safe_to_release=lambda: False) as lease:
|
||||
pass
|
||||
marker = locks.blocker_path(library)
|
||||
entered, resume, returned = Event(), Event(), Event()
|
||||
read = Path.read_text
|
||||
def withheld(path, *args, **kwargs):
|
||||
if path == marker:
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
returned.set()
|
||||
return read(path, *args, **kwargs)
|
||||
monkeypatch.setattr(Path, 'read_text', withheld)
|
||||
monkeypatch.setattr(win32process, 'CreateProcess', lambda *a, **k: pytest.fail('late native launch'))
|
||||
context = BatchTaskContext(library=str(library), targets=('A',))
|
||||
events = []
|
||||
try:
|
||||
result = run_verify(VerifyRequest(library, 'A', tmp_path/'evidence'), context=context,
|
||||
admission_timeout_seconds=.05, progress=events.append)
|
||||
assert entered.is_set() and result.status == 'TIMEOUT'
|
||||
assert not result.safe_to_continue
|
||||
assert result.release_proof['resource_scope']['operation_ids'] == [context.operation_id]
|
||||
def unrelated():
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0
|
||||
try:
|
||||
assert acquired
|
||||
return 'B completed'
|
||||
finally:
|
||||
if acquired: win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
assert NativeQueue().run(unrelated) == 'B completed'
|
||||
count = len(events)
|
||||
resume.set()
|
||||
assert returned.wait(2)
|
||||
limit = time.monotonic()+2
|
||||
while context.snapshot()['pending'] and time.monotonic() < limit:
|
||||
time.sleep(.01)
|
||||
assert context.snapshot()['pending'] == 0
|
||||
assert len(events) == count
|
||||
finally:
|
||||
resume.set()
|
||||
context.close(timeout_seconds=2)
|
||||
with locks._guard:
|
||||
owner = locks._owners.pop(lease.key, None)
|
||||
if owner: owner['handle'].Close()
|
||||
|
||||
|
||||
def test_native_time_does_not_consume_next_metadata_phase_budget(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, native_phase
|
||||
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
|
||||
stages = []
|
||||
def action():
|
||||
with native_phase():
|
||||
time.sleep(.12)
|
||||
stages.append('saved')
|
||||
time.sleep(.02)
|
||||
with native_phase():
|
||||
stages.append('closed')
|
||||
return stages
|
||||
assert context.run(action, timeout_seconds=.06) == ['saved', 'closed']
|
||||
|
||||
|
||||
def test_native_channel_enforces_cross_thread_serialization_and_reentrancy():
|
||||
from cbb_refresh_verify.operation_queue import native_channel
|
||||
with native_channel():
|
||||
with native_channel():
|
||||
with ThreadPoolExecutor(1) as pool:
|
||||
def contend():
|
||||
with native_channel():
|
||||
pytest.fail('concurrent native writer admitted')
|
||||
with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'):
|
||||
pool.submit(contend).result(2)
|
||||
with native_channel():
|
||||
pass
|
||||
|
||||
|
||||
def test_maintenance_checkpoint_admission_does_not_hold_native_channel(tmp_path, monkeypatch):
|
||||
from contextlib import nullcontext
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
with locks.LibraryLock(library, safe_to_release=lambda: False) as retained:
|
||||
pass
|
||||
marker = locks.blocker_path(library)
|
||||
entered, resume = Event(), Event()
|
||||
original = Path.read_bytes
|
||||
def withheld(path):
|
||||
if path == marker and not resume.is_set():
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
return original(path)
|
||||
monkeypatch.setattr(Path, 'read_bytes', withheld)
|
||||
def admit():
|
||||
with locks.LibraryLock.for_maintenance(library, tmp_path/'archive', lambda: nullcontext({})):
|
||||
pass
|
||||
with ThreadPoolExecutor(1) as pool:
|
||||
task = pool.submit(admit)
|
||||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||||
acquired = False
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
acquired = win32event.WaitForSingleObject(mutex, 0) == win32con.WAIT_OBJECT_0
|
||||
assert acquired
|
||||
finally:
|
||||
if acquired: win32event.ReleaseMutex(mutex)
|
||||
mutex.Close()
|
||||
resume.set()
|
||||
task.result(5)
|
||||
with locks._guard:
|
||||
owner = locks._owners.pop(retained.key, None)
|
||||
if owner: owner['handle'].Close()
|
||||
|
||||
|
||||
def test_native_handoff_keeps_real_mutex_until_bound_process_returns(tmp_path):
|
||||
import subprocess
|
||||
import sys
|
||||
from cbb_refresh_verify.operation_queue import (
|
||||
BatchTaskContext, NativeQueue, TaskHandoff, hold_native_handoff, native_channel)
|
||||
from cbb_refresh_verify.session_record import write_session_record
|
||||
process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
|
||||
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
|
||||
record = tmp_path/'session.json'
|
||||
write_session_record(record, dict(operation_id=context.operation_id, library=context.library,
|
||||
worker_pid=process.pid))
|
||||
queue = NativeQueue()
|
||||
effects = []
|
||||
def action():
|
||||
with native_channel():
|
||||
hold_native_handoff(process, record)
|
||||
with native_channel():
|
||||
effects.append('late mutation')
|
||||
try:
|
||||
with pytest.raises(TaskHandoff):
|
||||
queue.run(action, context=context)
|
||||
assert process.poll() is None
|
||||
assert context.snapshot()['pending_native']
|
||||
assert queue.run(lambda: 'pure read') == 'pure read'
|
||||
with pytest.raises(RuntimeError, match='NATIVE_CHANNEL_BUSY'):
|
||||
with native_channel():
|
||||
pytest.fail('writer overlapped handed-off native call')
|
||||
process.communicate('\n', timeout=3)
|
||||
limit = time.monotonic()+3
|
||||
while context.snapshot()['pending'] and time.monotonic() < limit:
|
||||
time.sleep(.01)
|
||||
assert context.snapshot()['pending'] == 0
|
||||
with native_channel():
|
||||
pass
|
||||
assert effects == []
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.communicate('\n', timeout=3)
|
||||
context.close(timeout_seconds=3)
|
||||
|
||||
|
||||
def test_worker_write_barrier_survives_supervisor_process_exit(tmp_path):
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from cbb_refresh_verify.operation_queue import native_channel
|
||||
ready, finish, done = (tmp_path/name for name in ('ready', 'finish', 'done'))
|
||||
child = '''import sys,time,os
|
||||
from pathlib import Path
|
||||
from cbb_refresh_verify.desktop_worker import worker_native_channel
|
||||
with worker_native_channel():
|
||||
Path(sys.argv[1]).write_text(str(os.getpid()))
|
||||
while not Path(sys.argv[2]).exists(): time.sleep(.02)
|
||||
Path(sys.argv[3]).touch()
|
||||
'''
|
||||
parent = '''import subprocess,sys,time,json
|
||||
from pathlib import Path
|
||||
subprocess.Popen([sys.executable,'-c',sys.argv[1],*sys.argv[2:]])
|
||||
deadline=time.monotonic()+4
|
||||
while not Path(sys.argv[2]).exists() and time.monotonic()<deadline: time.sleep(.02)
|
||||
raise SystemExit(0 if Path(sys.argv[2]).exists() else 2)
|
||||
'''
|
||||
supervisor = subprocess.Popen([sys.executable, '-c', parent, child, str(ready), str(finish), str(done)])
|
||||
try:
|
||||
assert supervisor.wait(timeout=6) == 0
|
||||
assert ready.is_file() and not done.exists()
|
||||
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
||||
with native_channel():
|
||||
pytest.fail('restart admitted a second writer')
|
||||
finally:
|
||||
finish.touch()
|
||||
deadline = time.monotonic()+3
|
||||
while not done.exists() and time.monotonic()<deadline: time.sleep(.02)
|
||||
supervisor.wait(timeout=3)
|
||||
assert done.is_file()
|
||||
with native_channel():
|
||||
pass
|
||||
|
||||
|
||||
def test_exact_sidecar_handle_survives_parent_exit_until_worker_boundary(tmp_path):
|
||||
import subprocess
|
||||
import sys
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
ready, finish, done, handle = (tmp_path/name for name in ('ready', 'finish', 'done', 'handle'))
|
||||
child = '''import sys,time
|
||||
from pathlib import Path
|
||||
from cbb_refresh_verify.desktop_worker import worker_native_channel
|
||||
while not Path(sys.argv[4]).exists(): time.sleep(.02)
|
||||
with worker_native_channel(lease_handle=int(Path(sys.argv[4]).read_text())):
|
||||
Path(sys.argv[1]).touch()
|
||||
while not Path(sys.argv[2]).exists(): time.sleep(.02)
|
||||
Path(sys.argv[3]).touch()
|
||||
'''
|
||||
parent = '''import subprocess,sys,time,win32api,win32con
|
||||
from pathlib import Path
|
||||
from cbb_refresh_verify.library_lock import LibraryLock,duplicate_worker_lease
|
||||
with LibraryLock(sys.argv[1],checkpoint=False):
|
||||
child=subprocess.Popen([sys.executable,'-c',sys.argv[2],*sys.argv[3:]])
|
||||
process=win32api.OpenProcess(win32con.PROCESS_DUP_HANDLE,False,child.pid)
|
||||
try: handle=duplicate_worker_lease(sys.argv[1],process)
|
||||
finally: process.Close()
|
||||
Path(sys.argv[6]).write_text(str(handle))
|
||||
deadline=time.monotonic()+4
|
||||
while not Path(sys.argv[3]).exists() and time.monotonic()<deadline: time.sleep(.02)
|
||||
assert Path(sys.argv[3]).exists()
|
||||
'''
|
||||
supervisor = subprocess.Popen([sys.executable, '-c', parent, str(library), child,
|
||||
str(ready), str(finish), str(done), str(handle)])
|
||||
try:
|
||||
assert supervisor.wait(timeout=6) == 0
|
||||
with pytest.raises(locks.LibraryBusy):
|
||||
with locks.LibraryLock(library, checkpoint=False):
|
||||
pytest.fail('uploader could replace a live worker library')
|
||||
finally:
|
||||
finish.touch()
|
||||
deadline = time.monotonic()+3
|
||||
while ready.exists() and not done.exists() and time.monotonic()<deadline: time.sleep(.02)
|
||||
supervisor.wait(timeout=3)
|
||||
assert done.is_file()
|
||||
with locks.LibraryLock(library, checkpoint=False):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize('obstacle', ['', 'unknown_title', 'duplicate', 'changed'])
|
||||
def test_failed_startup_dismisses_only_exact_stable_owned_tip(tmp_path, monkeypatch, obstacle):
|
||||
import win32gui
|
||||
from test_failed_startup import startup_fixture
|
||||
from cbb_refresh_verify.failed_session import close_failed_startup
|
||||
job, records, messages = startup_fixture(tmp_path, monkeypatch)
|
||||
visible = {100, 200}
|
||||
reads = []
|
||||
classes = {100: 'Afx:main', 200: '#32770', 201: 'Button', 202: 'Button'}
|
||||
def title(hwnd):
|
||||
if hwnd == 100: return 'Expedition PCB'
|
||||
if hwnd == 200:
|
||||
reads.append(hwnd)
|
||||
return 'Unknown' if obstacle == 'unknown_title' or obstacle == 'changed' and len(reads)>1 else 'Tip of the Day'
|
||||
return 'OK'
|
||||
monkeypatch.setattr(win32gui, 'EnumWindows', lambda fn, arg: [fn(h, arg) for h in sorted(visible)])
|
||||
monkeypatch.setattr(win32gui, 'EnumChildWindows', lambda hwnd, fn, arg:
|
||||
[fn(h, arg) for h in ((201, 202) if obstacle == 'duplicate' else (201,))] if hwnd == 200 else None)
|
||||
monkeypatch.setattr(win32gui, 'GetClassName', classes.__getitem__)
|
||||
monkeypatch.setattr(win32gui, 'GetWindowText', title)
|
||||
monkeypatch.setattr(win32gui, 'GetDlgCtrlID', lambda hwnd: 1)
|
||||
def post(hwnd, message, wparam, lparam):
|
||||
messages.append((hwnd, message))
|
||||
if hwnd == 201: visible.remove(200)
|
||||
elif hwnd == 100: records.clear()
|
||||
monkeypatch.setattr(win32gui, 'PostMessage', post)
|
||||
result = close_failed_startup(job, 12, timeout_seconds=0)
|
||||
if obstacle:
|
||||
assert result['status'] == 'RETAINED' and not messages
|
||||
else:
|
||||
assert result['status'] == 'CLOSED'
|
||||
assert messages == [(201, win32con.BM_CLICK), (100, win32con.WM_CLOSE)]
|
||||
|
||||
|
||||
def test_late_release_recheck_is_bounded_and_keeps_business_failure(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify.operation_lifecycle import recheck_failed_release
|
||||
from cbb_refresh_verify.verify_runner import VerifyResult
|
||||
from cbb_refresh_verify.session_record import write_session_record
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
evidence = tmp_path/'evidence'
|
||||
evidence.mkdir()
|
||||
result = VerifyResult('FAIL', stderr='native business failure', safe_to_continue=False,
|
||||
evidence_path=evidence/'result.json')
|
||||
result.evidence_path.write_text('{}')
|
||||
record = evidence/'session.json'
|
||||
write_session_record(record, dict(operation_id='failed', library=str(library), block='A', processes=[]))
|
||||
entered, resume = Event(), Event()
|
||||
original = Path.open
|
||||
def withheld(path, *args, **kwargs):
|
||||
if path == record:
|
||||
entered.set()
|
||||
assert resume.wait(3)
|
||||
return original(path, *args, **kwargs)
|
||||
monkeypatch.setattr(Path, 'open', withheld)
|
||||
try:
|
||||
started = time.monotonic()
|
||||
returned = recheck_failed_release(library, 'A', result, timeout_seconds=.05)
|
||||
assert time.monotonic()-started < .8
|
||||
assert entered.is_set() and returned is result
|
||||
assert returned.status == 'FAIL' and not returned.safe_to_continue
|
||||
finally:
|
||||
resume.set()
|
||||
|
||||
|
||||
def test_batch_context_cancel_records_all_remaining_items_and_new_batch_can_run(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext
|
||||
from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch
|
||||
requests = [VerifyRequest(tmp_path/'lib.lmc', name, tmp_path/'evidence') for name in ('A', 'B', 'C')]
|
||||
first = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A', 'B', 'C'))
|
||||
events = []
|
||||
def verify(request):
|
||||
first.cancel()
|
||||
return VerifyResult('CANCELLED')
|
||||
results = run_batch(requests, verifier=verify, context=first,
|
||||
progress=lambda name, result: events.append((name, result.status)))
|
||||
assert [r.status for r in results] == ['CANCELLED', 'SKIPPED', 'SKIPPED']
|
||||
assert events == [('A', 'CANCELLED'), ('B', 'SKIPPED'), ('C', 'SKIPPED')]
|
||||
second = BatchTaskContext(library=str(requests[0].lmc_file), targets=('A',))
|
||||
assert run_batch(requests[:1], verifier=lambda _: VerifyResult('PASS'), context=second)[0].status == 'PASS'
|
||||
|
||||
|
||||
def test_batch_retention_matches_normalized_library_alias_not_business_failure(tmp_path):
|
||||
from cbb_refresh_verify.verify_runner import VerifyRequest, VerifyResult, run_batch
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
alias = tmp_path/'A/sub/../lib.lmc'
|
||||
requests = [VerifyRequest(path, block, tmp_path/'evidence') for path, block in (
|
||||
(library, 'A'), (alias, 'B'), (tmp_path/'B/lib.lmc', 'C'), (tmp_path/'B/lib.lmc', 'D'))]
|
||||
calls = []
|
||||
def verify(request):
|
||||
calls.append(request.block_name)
|
||||
return VerifyResult('FAIL', safe_to_continue=request.block_name != 'A')
|
||||
results = run_batch(requests, verifier=verify)
|
||||
assert calls == ['A', 'C', 'D']
|
||||
assert [r.status for r in results] == ['FAIL', 'SKIPPED', 'FAIL', 'FAIL']
|
||||
|
||||
|
||||
def test_queue_close_has_one_deadline_and_reports_pending_scope(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue, TaskHandoff
|
||||
queue = NativeQueue()
|
||||
context = BatchTaskContext(library=str(tmp_path/'A.lmc'), aliases=('Z:/A.lmc',))
|
||||
entered, resume, finished = Event(), Event(), Event()
|
||||
def action():
|
||||
entered.set()
|
||||
resume.wait(3)
|
||||
def run():
|
||||
try:
|
||||
queue.run(action, context=context)
|
||||
except TaskHandoff:
|
||||
pass
|
||||
finally:
|
||||
finished.set()
|
||||
thread = Thread(target=run)
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
started = time.monotonic()
|
||||
states = queue.close(timeout_seconds=.05)
|
||||
assert time.monotonic()-started < .5
|
||||
assert states[0]['resource_scope']['aliases'] == ['Z:/A.lmc']
|
||||
assert states[0]['pending'] == 1
|
||||
assert finished.wait(1)
|
||||
finally:
|
||||
resume.set()
|
||||
thread.join(2)
|
||||
|
||||
|
||||
def test_real_management_transaction_does_not_handoff_between_remove_and_add(management_setup):
|
||||
from dataclasses import replace
|
||||
from test_management import execute
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
|
||||
request, adapter, source = management_setup
|
||||
original = adapter.remove
|
||||
def slow_remove(*args):
|
||||
result = original(*args)
|
||||
time.sleep(.4)
|
||||
return result
|
||||
adapter.remove = slow_remove # Native external boundary; transaction stays real.
|
||||
context = BatchTaskContext(library=str(request.library), targets=('Block',))
|
||||
result = NativeQueue().run(lambda: execute(replace(request, operation='update',
|
||||
source=source, auto_verify=False), adapter), context=context, timeout_seconds=.2)
|
||||
assert result.status == 'UPLOADED'
|
||||
assert adapter.calls[0] == 'remove' and adapter.calls[1][0] == 'add'
|
||||
|
||||
|
||||
def test_busy_native_channel_before_dispatch_does_not_retain_management_library(management_setup):
|
||||
from test_management import execute
|
||||
from cbb_refresh_verify.operation_queue import native_channel
|
||||
request, adapter, _ = management_setup
|
||||
entered, resume = Event(), Event()
|
||||
def owner():
|
||||
with native_channel():
|
||||
entered.set()
|
||||
resume.wait(3)
|
||||
thread = Thread(target=owner)
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
result = execute(request, adapter)
|
||||
assert result.safe_to_continue and not adapter.calls
|
||||
with locks.LibraryLock(request.library):
|
||||
pass
|
||||
finally:
|
||||
resume.set()
|
||||
thread.join(3)
|
||||
|
||||
|
||||
def test_only_explicit_repair_inspection_can_enter_while_worker_owns_writer_barrier(tmp_path):
|
||||
import json
|
||||
import os
|
||||
from cbb_refresh_verify.desktop_worker import worker_native_channel, run_native_worker
|
||||
from cbb_refresh_verify.operation_queue import native_channel
|
||||
from cbb_refresh_verify.native_repair_ui import repair_worker_main
|
||||
entered, resume = Event(), Event()
|
||||
def owner():
|
||||
with worker_native_channel():
|
||||
entered.set()
|
||||
resume.wait(3)
|
||||
thread = Thread(target=owner)
|
||||
thread.start()
|
||||
job = dict(root=str(tmp_path), project=str(tmp_path/'A.prj'), database=str(tmp_path/'database'),
|
||||
cancel_signal=str(tmp_path/'cancel'), supervisor_pid=os.getpid(), desktop='not-private')
|
||||
path = tmp_path/'repair-request.json'
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
with native_channel(inspection_only=True):
|
||||
pass
|
||||
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
||||
with native_channel():
|
||||
pass
|
||||
for mode in (False, 'true'):
|
||||
path.write_text(json.dumps({**job, 'inspection_only': mode}))
|
||||
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
||||
run_native_worker(repair_worker_main, path)
|
||||
path.write_text(json.dumps({**job, 'inspection_only': True}))
|
||||
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
||||
run_native_worker(lambda _: pytest.fail('arbitrary worker exempted'), path)
|
||||
assert run_native_worker(repair_worker_main, path) == 1
|
||||
result = json.loads((tmp_path/'repair-result.json').read_text())
|
||||
assert result['native_started'] is False
|
||||
assert 'REPAIR_PRIVATE_DESKTOP_REQUIRED' in result['detail']
|
||||
finally:
|
||||
resume.set()
|
||||
thread.join(3)
|
||||
|
||||
|
||||
def test_inspection_session_does_not_take_over_retained_writer_lease(tmp_path):
|
||||
import os
|
||||
import win32api
|
||||
from cbb_refresh_verify.desktop_worker import DesktopWorker
|
||||
library = tmp_path/'A/lib.lmc'
|
||||
library.parent.mkdir()
|
||||
library.touch()
|
||||
with locks.LibraryLock(library, safe_to_release=lambda: False) as lease:
|
||||
pass
|
||||
worker = object.__new__(DesktopWorker)
|
||||
worker.handle, worker.pid = win32api.GetCurrentProcess(), os.getpid()
|
||||
try:
|
||||
worker.bind_session(dict(operation_id='inspection', operation='inspect', inspection_only=True,
|
||||
library=str(library), project=str(tmp_path/'A.prj'), session_record_path=str(tmp_path/'session.json')))
|
||||
assert locks.has_retained_library(library)
|
||||
from cbb_refresh_verify.session_record import read_session_record
|
||||
record = read_session_record(tmp_path/'session.json', evidence_root=tmp_path)
|
||||
assert record['worker_lease_handle'] is None
|
||||
finally:
|
||||
with locks._guard:
|
||||
owner = locks._owners.pop(lease.key, None)
|
||||
if owner: owner['handle'].Close()
|
||||
|
||||
|
||||
def test_handoff_preserves_known_business_result_and_save_facts(tmp_path):
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, remember_result
|
||||
from cbb_refresh_verify.verify_runner import VerifyResult
|
||||
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
|
||||
native_result = VerifyResult('FAIL', stderr='native failure after Save', safe_to_continue=False,
|
||||
write_outcome=dict(save_state='COMMITTED', close_state='UNKNOWN'))
|
||||
resume, done = Event(), Event()
|
||||
def finalize():
|
||||
remember_result(native_result)
|
||||
try:
|
||||
resume.wait(3)
|
||||
finally:
|
||||
done.set()
|
||||
try:
|
||||
with pytest.raises(TaskHandoff) as error:
|
||||
context.run(finalize, timeout_seconds=.05)
|
||||
assert error.value.result is native_result
|
||||
assert error.value.result.write_outcome['save_state'] == 'COMMITTED'
|
||||
assert error.value.result.status == 'FAIL'
|
||||
from cbb_refresh_verify.verify_runner import _handoff_verify_result
|
||||
returned = _handoff_verify_result(error.value)
|
||||
assert returned.status == 'FAIL' and returned.stderr == native_result.stderr
|
||||
assert returned.write_outcome == native_result.write_outcome and not returned.safe_to_continue
|
||||
error.value.result = VerifyResult('PASS', write_outcome=native_result.write_outcome)
|
||||
unfinished = _handoff_verify_result(error.value)
|
||||
assert unfinished.status == 'FAIL' and unfinished.library_flag == 'UNKNOWN'
|
||||
assert not unfinished.safe_to_continue
|
||||
finally:
|
||||
resume.set()
|
||||
assert done.wait(2)
|
||||
|
||||
|
||||
def test_registered_writer_owns_process_handle_after_callers_handle_closes(monkeypatch):
|
||||
import subprocess
|
||||
import sys
|
||||
from cbb_refresh_verify import desktop_worker
|
||||
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
||||
process = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
|
||||
entered, done = Event(), Event()
|
||||
errors = []
|
||||
def run():
|
||||
try:
|
||||
with desktop_worker.worker_native_channel():
|
||||
desktop_worker.register_native_writer(process)
|
||||
process._handle.Close()
|
||||
entered.set()
|
||||
except Exception as error:
|
||||
errors.append(error)
|
||||
finally:
|
||||
done.set()
|
||||
thread = Thread(target=run)
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(2)
|
||||
assert not done.wait(.1), 'a closed caller handle released an unreturned writer'
|
||||
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
||||
desktop_worker.assert_native_writer_idle()
|
||||
finally:
|
||||
process.stdin.write('\n')
|
||||
process.stdin.flush()
|
||||
process.stdin.close()
|
||||
thread.join(3)
|
||||
# The duplicate's observed exit is authoritative; the original Popen
|
||||
# handle was deliberately invalidated, so do not poll it in __del__.
|
||||
process.returncode = 0
|
||||
assert done.is_set() and not errors
|
||||
@@ -17,7 +17,9 @@ def test_batch_rechecks_owned_failure_before_skipping_next_item(tmp_path,monkeyp
|
||||
monkeypatch.setattr(session_record,'inspect_session_record',lambda *a,**k:dict(status='UNCONFIRMED'))
|
||||
def proof(*a,**k):
|
||||
if not released:raise library_lock.LibraryBusy('shared database still occupied')
|
||||
return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False)
|
||||
return dict(status='RELEASE_CONFIRMED',library=str(library),data_recovery_confirmed=False,
|
||||
lease_state='RELEASED',blocked_resources=[],
|
||||
resource_scope=dict(scope_state='OBSERVED',library=str(library)))
|
||||
monkeypatch.setattr(library_lock,'release_retained_library',proof)
|
||||
def verify(req):
|
||||
calls.append(req.block_name)
|
||||
|
||||
@@ -215,7 +215,7 @@ def test_cancel_and_restart_never_replay(setup):
|
||||
from cbb_refresh_verify.management import run_management_batch, unfinished_operations
|
||||
request, adapter, _ = setup
|
||||
stop = Event(); stop.set()
|
||||
assert run_management_batch([request], stop=stop, adapter=adapter) == []
|
||||
assert [r.status for r in run_management_batch([request], stop=stop, adapter=adapter)] == ['SKIPPED']
|
||||
checkpoint = request.evidence_dir/'interrupted/checkpoint.json'
|
||||
checkpoint.parent.mkdir(parents=True)
|
||||
checkpoint.write_text('{"stage":"native_remove", "status":"RUNNING"}', encoding='utf-8')
|
||||
|
||||
@@ -313,8 +313,12 @@ def test_private_worker_timeout_retains_native_and_never_kills(tmp_path, monkeyp
|
||||
def test_hidden_worker_and_probe_dispatch_before_qt(monkeypatch, tmp_path):
|
||||
import cbb_refresh_verify.native_management as native
|
||||
from cbb_refresh_verify.application import main
|
||||
from cbb_refresh_verify import desktop_worker
|
||||
# This dispatch represents a new worker process, not other tests' mocked clients.
|
||||
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
||||
seen = []
|
||||
monkeypatch.setattr(native, 'management_worker_main', lambda p: (seen.append(p) or 17))
|
||||
(tmp_path/'job.json').write_text('{}')
|
||||
assert main(['--management-worker', str(tmp_path/'job.json')]) == 17
|
||||
assert seen == [tmp_path/'job.json']
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ def test_user_can_close_unknown_client_before_confirming(tmp_path,monkeypatch):
|
||||
_idle_check(*paths,confirmation=WriteAdmission(prompt=prompt),evidence=tmp_path/'evidence',block='Block')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED'])])
|
||||
@pytest.mark.parametrize('first,expected',[('skip',['SKIPPED','UPLOADED']),('cancel',['CANCELLED','SKIPPED'])])
|
||||
def test_batch_continues_only_after_skip_or_confirmed_release(tmp_path,monkeypatch,first,expected):
|
||||
from dataclasses import replace
|
||||
from test_management import FileAdapter,project
|
||||
@@ -149,7 +149,7 @@ def test_cancel_batch_also_stops_default_api_without_a_caller_event(tmp_path,mon
|
||||
request=ManagementRequest('add',library,'One',source=source,auto_verify=False,evidence_dir=tmp_path/'evidence')
|
||||
adapter=FileAdapter()
|
||||
results=run_management_batch([request,replace(request,name='Two')],adapter=adapter,version_prompt=prompt)
|
||||
assert [r.status for r in results]==['CANCELLED']
|
||||
assert [r.status for r in results]==['CANCELLED','SKIPPED']
|
||||
assert len(choices)==1 and not adapter.calls
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user