From 07206ce0d1ac003f42468231cbad0a36cb74db36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Tue, 17 Nov 2026 00:39:30 +0800 Subject: [PATCH] fix: preserve native reads and advancing metadata operations --- src/cbb_refresh_verify/native_management.py | 18 +++--- src/cbb_refresh_verify/operation_queue.py | 10 ++++ src/cbb_refresh_verify/scoped_manifest.py | 8 ++- tests/test_issue15_manifest_progress.py | 65 +++++++++++++++++++++ tests/test_issue15_worker_admission.py | 35 +++++++++++ 5 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 tests/test_issue15_manifest_progress.py diff --git a/src/cbb_refresh_verify/native_management.py b/src/cbb_refresh_verify/native_management.py index 3a07d2b..3d64011 100644 --- a/src/cbb_refresh_verify/native_management.py +++ b/src/cbb_refresh_verify/native_management.py @@ -244,13 +244,17 @@ class NativeManagementAdapter: worker.bind_cancellation(job['cancel_signal']) if hasattr(worker, 'bind_session'): worker.bind_session(job) - from .operation_queue import commit_worker_start - commit_worker_start(worker, job) - while worker.poll() is None and time.monotonic() < deadline: - if self.stop is not None and self.stop.is_set(): - Path(job['cancel_signal']).touch() - break - time.sleep(.1) + from contextlib import nullcontext + from .operation_queue import commit_worker_start, native_channel + # Only admitted native waiting outlives the UI's metadata deadline. + channel = native_channel(read_only=True) if job.get('worker_role') == 'catalog_readonly' else nullcontext() + with channel: + commit_worker_start(worker, job) + while worker.poll() is None and time.monotonic() < deadline: + if self.stop is not None and self.stop.is_set(): + Path(job['cancel_signal']).touch() + break + time.sleep(.1) if worker.poll() is None: # DesktopWorker.close() requests a kill without a cancellation file. # Keep this exact worker alive until its native call returns normally. diff --git a/src/cbb_refresh_verify/operation_queue.py b/src/cbb_refresh_verify/operation_queue.py index c339b46..3df0684 100644 --- a/src/cbb_refresh_verify/operation_queue.py +++ b/src/cbb_refresh_verify/operation_queue.py @@ -99,6 +99,16 @@ def check_active(): raise QueueCancelled('任务已取消,未提交新的原生操作。') +def note_metadata_progress(): + """Renew inactivity only after completed I/O; never revive an abandoned call.""" + call = getattr(_local, 'call', None) + if call is not None: + with _tasks_guard: + if not call.native_depth: + check_active() + call.phase_started = time.monotonic() + + def remember_result(result): call = getattr(_local, 'call', None) if call is not None: diff --git a/src/cbb_refresh_verify/scoped_manifest.py b/src/cbb_refresh_verify/scoped_manifest.py index 9df40c9..9339357 100644 --- a/src/cbb_refresh_verify/scoped_manifest.py +++ b/src/cbb_refresh_verify/scoped_manifest.py @@ -12,6 +12,7 @@ import os from pathlib import Path import shutil import stat +from .operation_queue import note_metadata_progress MANIFEST_VERSION = 1 @@ -207,6 +208,7 @@ def scan_tree(root, *, scope=None, exclude=None, check=None): if not (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode)): raise ValueError(f'不支持的特殊文件:{entry.path}') entries[path] = metadata + note_metadata_progress() if stat.S_ISDIR(metadata.st_mode): folders.append(path) return dict(sorted(entries.items())) @@ -252,6 +254,7 @@ def build_manifest(root, *, scope=None, exclude=None, progress=None, hash_functi _emit(progress, 'hash', 0, 0, total_files=len(selected)) with ThreadPoolExecutor(max_workers=8) as pool: for count, (relative, item) in enumerate(pool.map(hashed, selected), 1): + note_metadata_progress() files[relative] = item hashed_bytes += item['size'] if count % 256 == 0 or count == len(selected): @@ -285,6 +288,7 @@ def build_exclusive_library_manifest(root, *, scope, progress=None): raise OSError(f'LIBRARY_DATABASE_IN_USE: 无法独占读取 {path}:{error}') from error handles[path] = handle stack.callback(handle.Close) + note_metadata_progress() def digest(path): if path not in handles: @@ -388,6 +392,7 @@ def copy_verified(source, destination, *, scope=None, progress=None, hash_functi with ThreadPoolExecutor(max_workers=8) as pool: for count, (relative, digest, label) in enumerate(pool.map(copy_file, files), 1): + note_metadata_progress() if label is not None: size = entries[source/relative].st_size record[relative] = {'size': size, 'sha256': digest, 'scope': label} @@ -405,7 +410,8 @@ def copy_verified(source, destination, *, scope=None, progress=None, hash_functi if hash_function(destination/relative) != item['sha256']: raise OSError(f'复制内容校验失败:{path}') with ThreadPoolExecutor(max_workers=8) as pool: - list(pool.map(validate_bytes, record)) + for _ in pool.map(validate_bytes, record): + note_metadata_progress() if _metadata_record(source, scan_tree(source)) != expected_metadata: raise OSError(f'复制期间目录内容变化:{source}') if _metadata_record(destination, scan_tree(destination)) != expected_metadata: diff --git a/tests/test_issue15_manifest_progress.py b/tests/test_issue15_manifest_progress.py new file mode 100644 index 0000000..f12ad2c --- /dev/null +++ b/tests/test_issue15_manifest_progress.py @@ -0,0 +1,65 @@ +"""Large advancing manifests stay usable; a stalled member still hands off.""" +from threading import Event +import time + +import pytest + +from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, check_active +from cbb_refresh_verify.scoped_manifest import build_manifest, hash_file + + +def test_advancing_manifest_outlives_aggregate_metadata_deadline(tmp_path, monkeypatch): + for i in range(768): + (tmp_path/f'part-{i:04}.dat').write_bytes(str(i).encode()) + clock, offset = time.monotonic, [0] + monkeypatch.setattr(time, 'monotonic', lambda:clock()+offset[0]) + def progress(event): + if event['hashed_files']: + offset[0] += 8 + time.sleep(.06) # Let the actual context waiter observe each advancing batch. + context = BatchTaskContext(str(tmp_path/'library.lmc'), native_policy='external') + try: + result = context.run(lambda:build_manifest(tmp_path, progress=progress), timeout_seconds=10) + assert len(result['files']) == 768 + assert offset[0] == 24 + finally: + context.close(timeout_seconds=2) + + +def test_stalled_manifest_member_cannot_resume_into_native_write(tmp_path): + (tmp_path/'one.dat').write_bytes(b'content') + entered, release = Event(), Event() + writes = [] + def stalled(path): + entered.set() + assert release.wait(3) + return hash_file(path) + def action(): + build_manifest(tmp_path, hash_function=stalled) + check_active() + writes.append('native') + context = BatchTaskContext(str(tmp_path/'library.lmc'), native_policy='external') + try: + with pytest.raises(TaskHandoff): + context.run(action, timeout_seconds=.1) + assert entered.is_set() + release.set() + context.close(timeout_seconds=2) + assert writes == [] + finally: + release.set() + + +def test_manifest_progress_does_not_interrupt_admitted_native_continuation(tmp_path): + from cbb_refresh_verify.operation_queue import native_phase + (tmp_path/'one.dat').write_bytes(b'content') + context = BatchTaskContext(str(tmp_path/'library.lmc')) + effects = [] + def transaction(): + with native_phase(): + effects.append('Remove') + context.cancel() + assert len(build_manifest(tmp_path)['files']) == 1 + effects.append('Add') + context.run(transaction) + assert effects == ['Remove', 'Add'] diff --git a/tests/test_issue15_worker_admission.py b/tests/test_issue15_worker_admission.py index 87ad2d4..a12b25c 100644 --- a/tests/test_issue15_worker_admission.py +++ b/tests/test_issue15_worker_admission.py @@ -203,3 +203,38 @@ def test_catalog_read_role_requires_the_actual_readonly_worker_action(tmp_path): assert effects==[] assert catalog_dialog_action('Library Manager','Would you like to save?',read_only=True)=='cancel' assert catalog_dialog_action('Upgrade library','Continue?',read_only=True)=='cancel' + + +def test_catalog_native_wait_outlives_metadata_deadline(tmp_path, transport, monkeypatch): + from threading import Thread + from cbb_refresh_verify.native_management import NativeManagementAdapter + from cbb_refresh_verify.operation_queue import BatchTaskContext + Transport, workers = transport + evidence = tmp_path/'slow-catalog' + original = Transport.__init__ + def start(self, command, *args, **kwargs): + Path(command[-1]).with_name('hold-handler').touch() + original(self, command, *args, **kwargs) + Transport.__init__ = start + clock = time.monotonic + offset = [0] + monkeypatch.setattr(time, 'monotonic', lambda: clock()+offset[0]) + def complete(): + deadline = clock()+5 + while not (evidence/'handler-entered').exists() and clock()