Files
cbb-refresh-verify-795/tests/test_recovery_metadata_handoff.py

153 lines
6.8 KiB
Python

from threading import Event
from types import SimpleNamespace
import time
import pytest
from test_backup_recovery import recovery_files
from cbb_refresh_verify import backup_recovery as recovery
from cbb_refresh_verify.library_lock import LibraryLock,LibraryBusy
from cbb_refresh_verify.operation_queue import BatchTaskContext,TaskHandoff,native_channel
from cbb_refresh_verify.verify_runner import VerifyResult
@pytest.mark.parametrize('boundary',['idle','staging','final_manifest'])
@pytest.mark.parametrize('cancel',[False,True])
def test_real_recovery_handoff_preserves_A_lease_without_blocking_B_or_late_install(recovery_files,monkeypatch,boundary,cancel):
request,original,source=recovery_files
entered,release,done=Event(),Event(),Event();calls=[]
context=BatchTaskContext(str(request.lmc_file),native_policy='external')
from cbb_refresh_verify import operation_queue
# Expire inactivity at the chosen stall, not during variable filesystem setup.
clock=[time.monotonic()]
monkeypatch.setattr(operation_queue,'time',SimpleNamespace(
monotonic=lambda:clock[0] if clock[0] is not None else time.monotonic()))
def hold():
entered.set()
if cancel:context.cancel()
clock[0]=time.monotonic()+1
assert release.wait(5)
def idle(*args):
if boundary=='idle' and not entered.is_set():hold()
copier=recovery._copy_verified
def copy(*args,**kwargs):
value=copier(*args,**kwargs)
if boundary=='staging' and not entered.is_set():hold()
return value
monkeypatch.setattr(recovery,'_copy_verified',copy)
manifest=recovery._manifest
def build(root,**kwargs):
value=manifest(root,**kwargs)
if boundary=='final_manifest' and root==request.lmc_file.parent:hold()
return value
monkeypatch.setattr(recovery,'_manifest',build)
def verify(req,**kwargs):
calls.append(req.lmc_file)
return VerifyResult('PASS',library_flag='GREEN')
def action():
try:return recovery.recover_from_backup(request,source,verifier=verify,idle_check=idle)
finally:done.set()
refusal=None
try:
with pytest.raises(TaskHandoff):context.run(action,timeout_seconds=.3)
assert entered.is_set()
clock[0]=None
with pytest.raises(LibraryBusy):
with LibraryLock(request.lmc_file):pytest.fail('A lease lost during stalled metadata')
def B():
with native_channel():return 'admitted'
try:
result=BatchTaskContext(str(request.lmc_file.parent.parent/'B/library.lmc'),native_policy='external').run(B)
except RuntimeError as error:refusal=str(error)
finally:
release.set();assert done.wait(5);context.close(timeout_seconds=2)
assert (original.parent/'database/icdb.dat').read_bytes()==b'damaged'
assert not list(original.parent.parent.glob('Work.before-restore-*'))
assert len(calls)==(1 if boundary=='final_manifest' else 0)
assert request.lmc_file not in calls
assert refusal is None,refusal
assert result=='admitted'
def test_cancel_during_work_swap_finishes_atomic_pair_without_target_verify(recovery_files,monkeypatch):
from pathlib import Path
request,original,source=recovery_files
context=BatchTaskContext(str(request.lmc_file),native_policy='external');calls=[]
rename=Path.rename
def swap(path,target):
value=rename(path,target)
if path==original.parent:context.cancel()
return value
monkeypatch.setattr(Path,'rename',swap)
def verify(req,**kwargs):
calls.append(req.lmc_file)
return VerifyResult('PASS',library_flag='GREEN')
result=context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
assert result.status!='PASS'
assert (original.parent/'database/icdb.dat').read_bytes()==b'healthy'
retained=list(original.parent.parent.glob('Work.before-restore-*'))
assert len(retained)==1 and (retained[0]/'database/icdb.dat').read_bytes()==b'damaged'
assert len(calls)==1 and request.lmc_file not in calls
def test_shutdown_cannot_abandon_local_work_swap_or_hold_host_channel(recovery_files,monkeypatch):
from pathlib import Path
from threading import Thread
request,original,source=recovery_files
context=BatchTaskContext(str(request.lmc_file),native_policy='external')
entered,release,done=Event(),Event(),Event();calls=[];errors=[]
rename=Path.rename
def swap(path,target):
value=rename(path,target)
if path==original.parent:
entered.set();assert release.wait(5)
return value
monkeypatch.setattr(Path,'rename',swap)
def verify(req,**kwargs):
calls.append(req.lmc_file);return VerifyResult('PASS',library_flag='GREEN')
def run():
try:context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
except Exception as error:errors.append(error)
finally:done.set()
thread=Thread(target=run);thread.start()
try:
assert entered.wait(3)
state=context.shutdown()
assert state['shutdown_handoff'] is False
assert not done.wait(.1)
def B():
with native_channel():return True
assert BatchTaskContext(str(request.lmc_file.parent.parent/'B/library.lmc'),native_policy='external').run(B)
finally:
release.set();thread.join(5)
assert done.is_set() and len(calls)==1
assert (original.parent/'database/icdb.dat').read_bytes()==b'healthy'
def test_cancelled_failed_install_rolls_back_before_releasing_commit(recovery_files,monkeypatch):
from pathlib import Path
request,original,source=recovery_files;calls=[]
context=BatchTaskContext(str(request.lmc_file),native_policy='external');rename=Path.rename
def swap(path,target):
if path.name.startswith('Work.restore-'):
context.cancel();raise OSError('install rename refused')
return rename(path,target)
monkeypatch.setattr(Path,'rename',swap)
def verify(req,**kwargs):
calls.append(req.lmc_file);return VerifyResult('PASS',library_flag='GREEN')
result=context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
assert result.status!='PASS' and 'install rename refused' in result.stderr
assert (original.parent/'database/icdb.dat').read_bytes()==b'damaged'
assert not list(original.parent.parent.glob('Work.before-restore-*'))
assert len(calls)==1
def test_metadata_only_context_cannot_enter_local_commit(tmp_path):
from cbb_refresh_verify.operation_queue import local_commit
mutations=[]
def commit():
with local_commit():mutations.append('rename')
with pytest.raises(RuntimeError,match='Read-only metadata'):
BatchTaskContext(str(tmp_path/'lib.lmc')).run(commit,metadata_only=True)
assert not mutations