fix: preserve native reads and advancing metadata operations

This commit is contained in:
阿白叔 committed 2026-11-17 00:39:30 +08:00
1 parent 5b7e7bea2f
commit 07206ce0d1
5 files changed
+128 -8

No files matched your search

+11 -7
View File
@@ -244,13 +244,17 @@ class NativeManagementAdapter:
worker.bind_cancellation(job['cancel_signal'])
if hasattr(worker, 'bind_session'):
worker.bind_session(job)
from .operation_queue import commit_worker_start
commit_worker_start(worker, job)
while worker.poll() is None and time.monotonic() < deadline:
if self.stop is not None and self.stop.is_set():
Path(job['cancel_signal']).touch()
break
time.sleep(.1)
from contextlib import nullcontext
from .operation_queue import commit_worker_start, native_channel
# Only admitted native waiting outlives the UI's metadata deadline.
channel = native_channel(read_only=True) if job.get('worker_role') == 'catalog_readonly' else nullcontext()
with channel:
commit_worker_start(worker, job)
while worker.poll() is None and time.monotonic() < deadline:
if self.stop is not None and self.stop.is_set():
Path(job['cancel_signal']).touch()
break
time.sleep(.1)
if worker.poll() is None:
# DesktopWorker.close() requests a kill without a cancellation file.
# Keep this exact worker alive until its native call returns normally.
+10
View File
@@ -99,6 +99,16 @@ def check_active():
raise QueueCancelled('任务已取消,未提交新的原生操作。')
def note_metadata_progress():
"""Renew inactivity only after completed I/O; never revive an abandoned call."""
call = getattr(_local, 'call', None)
if call is not None:
with _tasks_guard:
if not call.native_depth:
check_active()
call.phase_started = time.monotonic()
def remember_result(result):
call = getattr(_local, 'call', None)
if call is not None:
+7 -1
View File
@@ -12,6 +12,7 @@ import os
from pathlib import Path
import shutil
import stat
from .operation_queue import note_metadata_progress
MANIFEST_VERSION = 1
@@ -207,6 +208,7 @@ def scan_tree(root, *, scope=None, exclude=None, check=None):
if not (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode)):
raise ValueError(f'不支持的特殊文件:{entry.path}')
entries[path] = metadata
note_metadata_progress()
if stat.S_ISDIR(metadata.st_mode):
folders.append(path)
return dict(sorted(entries.items()))
@@ -252,6 +254,7 @@ def build_manifest(root, *, scope=None, exclude=None, progress=None, hash_functi
_emit(progress, 'hash', 0, 0, total_files=len(selected))
with ThreadPoolExecutor(max_workers=8) as pool:
for count, (relative, item) in enumerate(pool.map(hashed, selected), 1):
note_metadata_progress()
files[relative] = item
hashed_bytes += item['size']
if count % 256 == 0 or count == len(selected):
@@ -285,6 +288,7 @@ def build_exclusive_library_manifest(root, *, scope, progress=None):
raise OSError(f'LIBRARY_DATABASE_IN_USE: 无法独占读取 {path}:{error}') from error
handles[path] = handle
stack.callback(handle.Close)
note_metadata_progress()
def digest(path):
if path not in handles:
@@ -388,6 +392,7 @@ def copy_verified(source, destination, *, scope=None, progress=None, hash_functi
with ThreadPoolExecutor(max_workers=8) as pool:
for count, (relative, digest, label) in enumerate(pool.map(copy_file, files), 1):
note_metadata_progress()
if label is not None:
size = entries[source/relative].st_size
record[relative] = {'size': size, 'sha256': digest, 'scope': label}
@@ -405,7 +410,8 @@ def copy_verified(source, destination, *, scope=None, progress=None, hash_functi
if hash_function(destination/relative) != item['sha256']:
raise OSError(f'复制内容校验失败:{path}')
with ThreadPoolExecutor(max_workers=8) as pool:
list(pool.map(validate_bytes, record))
for _ in pool.map(validate_bytes, record):
note_metadata_progress()
if _metadata_record(source, scan_tree(source)) != expected_metadata:
raise OSError(f'复制期间目录内容变化:{source}')
if _metadata_record(destination, scan_tree(destination)) != expected_metadata:
+65
View File
@@ -0,0 +1,65 @@
"""Large advancing manifests stay usable; a stalled member still hands off."""
from threading import Event
import time
import pytest
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, check_active
from cbb_refresh_verify.scoped_manifest import build_manifest, hash_file
def test_advancing_manifest_outlives_aggregate_metadata_deadline(tmp_path, monkeypatch):
for i in range(768):
(tmp_path/f'part-{i:04}.dat').write_bytes(str(i).encode())
clock, offset = time.monotonic, [0]
monkeypatch.setattr(time, 'monotonic', lambda:clock()+offset[0])
def progress(event):
if event['hashed_files']:
offset[0] += 8
time.sleep(.06) # Let the actual context waiter observe each advancing batch.
context = BatchTaskContext(str(tmp_path/'library.lmc'), native_policy='external')
try:
result = context.run(lambda:build_manifest(tmp_path, progress=progress), timeout_seconds=10)
assert len(result['files']) == 768
assert offset[0] == 24
finally:
context.close(timeout_seconds=2)
def test_stalled_manifest_member_cannot_resume_into_native_write(tmp_path):
(tmp_path/'one.dat').write_bytes(b'content')
entered, release = Event(), Event()
writes = []
def stalled(path):
entered.set()
assert release.wait(3)
return hash_file(path)
def action():
build_manifest(tmp_path, hash_function=stalled)
check_active()
writes.append('native')
context = BatchTaskContext(str(tmp_path/'library.lmc'), native_policy='external')
try:
with pytest.raises(TaskHandoff):
context.run(action, timeout_seconds=.1)
assert entered.is_set()
release.set()
context.close(timeout_seconds=2)
assert writes == []
finally:
release.set()
def test_manifest_progress_does_not_interrupt_admitted_native_continuation(tmp_path):
from cbb_refresh_verify.operation_queue import native_phase
(tmp_path/'one.dat').write_bytes(b'content')
context = BatchTaskContext(str(tmp_path/'library.lmc'))
effects = []
def transaction():
with native_phase():
effects.append('Remove')
context.cancel()
assert len(build_manifest(tmp_path)['files']) == 1
effects.append('Add')
context.run(transaction)
assert effects == ['Remove', 'Add']
+35
View File
@@ -203,3 +203,38 @@ def test_catalog_read_role_requires_the_actual_readonly_worker_action(tmp_path):
assert effects==[]
assert catalog_dialog_action('Library Manager','Would you like to save?',read_only=True)=='cancel'
assert catalog_dialog_action('Upgrade library','Continue?',read_only=True)=='cancel'
def test_catalog_native_wait_outlives_metadata_deadline(tmp_path, transport, monkeypatch):
from threading import Thread
from cbb_refresh_verify.native_management import NativeManagementAdapter
from cbb_refresh_verify.operation_queue import BatchTaskContext
Transport, workers = transport
evidence = tmp_path/'slow-catalog'
original = Transport.__init__
def start(self, command, *args, **kwargs):
Path(command[-1]).with_name('hold-handler').touch()
original(self, command, *args, **kwargs)
Transport.__init__ = start
clock = time.monotonic
offset = [0]
monkeypatch.setattr(time, 'monotonic', lambda: clock()+offset[0])
def complete():
deadline = clock()+5
while not (evidence/'handler-entered').exists() and clock()<deadline:
time.sleep(.01)
offset[0] = 20 # Only the admitted native wait crosses the metadata deadline.
time.sleep(.15)
(evidence/'finish').touch()
finisher = Thread(target=complete)
finisher.start()
context = BatchTaskContext(str(tmp_path/'library.lmc'), native_policy='external')
try:
result = context.run(lambda: NativeManagementAdapter(worker_factory=Transport).catalog(
tmp_path/'library.lmc', evidence), timeout_seconds=10)
assert [entry.name for entry in result.entries] == ['A']
assert result.safe_to_continue
finally:
(evidence/'finish').touch()
finisher.join(6)
context.close(timeout_seconds=6)