153 lines
6.8 KiB
Python
153 lines
6.8 KiB
Python
from threading import Event
|
|
from types import SimpleNamespace
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from test_backup_recovery import recovery_files
|
|
from cbb_refresh_verify import backup_recovery as recovery
|
|
from cbb_refresh_verify.library_lock import LibraryLock,LibraryBusy
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext,TaskHandoff,native_channel
|
|
from cbb_refresh_verify.verify_runner import VerifyResult
|
|
|
|
|
|
@pytest.mark.parametrize('boundary',['idle','staging','final_manifest'])
|
|
@pytest.mark.parametrize('cancel',[False,True])
|
|
def test_real_recovery_handoff_preserves_A_lease_without_blocking_B_or_late_install(recovery_files,monkeypatch,boundary,cancel):
|
|
request,original,source=recovery_files
|
|
entered,release,done=Event(),Event(),Event();calls=[]
|
|
context=BatchTaskContext(str(request.lmc_file),native_policy='external')
|
|
from cbb_refresh_verify import operation_queue
|
|
# Expire inactivity at the chosen stall, not during variable filesystem setup.
|
|
clock=[time.monotonic()]
|
|
monkeypatch.setattr(operation_queue,'time',SimpleNamespace(
|
|
monotonic=lambda:clock[0] if clock[0] is not None else time.monotonic()))
|
|
def hold():
|
|
entered.set()
|
|
if cancel:context.cancel()
|
|
clock[0]=time.monotonic()+1
|
|
assert release.wait(5)
|
|
def idle(*args):
|
|
if boundary=='idle' and not entered.is_set():hold()
|
|
copier=recovery._copy_verified
|
|
def copy(*args,**kwargs):
|
|
value=copier(*args,**kwargs)
|
|
if boundary=='staging' and not entered.is_set():hold()
|
|
return value
|
|
monkeypatch.setattr(recovery,'_copy_verified',copy)
|
|
manifest=recovery._manifest
|
|
def build(root,**kwargs):
|
|
value=manifest(root,**kwargs)
|
|
if boundary=='final_manifest' and root==request.lmc_file.parent:hold()
|
|
return value
|
|
monkeypatch.setattr(recovery,'_manifest',build)
|
|
def verify(req,**kwargs):
|
|
calls.append(req.lmc_file)
|
|
return VerifyResult('PASS',library_flag='GREEN')
|
|
def action():
|
|
try:return recovery.recover_from_backup(request,source,verifier=verify,idle_check=idle)
|
|
finally:done.set()
|
|
refusal=None
|
|
try:
|
|
with pytest.raises(TaskHandoff):context.run(action,timeout_seconds=.3)
|
|
assert entered.is_set()
|
|
clock[0]=None
|
|
with pytest.raises(LibraryBusy):
|
|
with LibraryLock(request.lmc_file):pytest.fail('A lease lost during stalled metadata')
|
|
def B():
|
|
with native_channel():return 'admitted'
|
|
try:
|
|
result=BatchTaskContext(str(request.lmc_file.parent.parent/'B/library.lmc'),native_policy='external').run(B)
|
|
except RuntimeError as error:refusal=str(error)
|
|
finally:
|
|
release.set();assert done.wait(5);context.close(timeout_seconds=2)
|
|
assert (original.parent/'database/icdb.dat').read_bytes()==b'damaged'
|
|
assert not list(original.parent.parent.glob('Work.before-restore-*'))
|
|
assert len(calls)==(1 if boundary=='final_manifest' else 0)
|
|
assert request.lmc_file not in calls
|
|
assert refusal is None,refusal
|
|
assert result=='admitted'
|
|
|
|
|
|
def test_cancel_during_work_swap_finishes_atomic_pair_without_target_verify(recovery_files,monkeypatch):
|
|
from pathlib import Path
|
|
request,original,source=recovery_files
|
|
context=BatchTaskContext(str(request.lmc_file),native_policy='external');calls=[]
|
|
rename=Path.rename
|
|
def swap(path,target):
|
|
value=rename(path,target)
|
|
if path==original.parent:context.cancel()
|
|
return value
|
|
monkeypatch.setattr(Path,'rename',swap)
|
|
def verify(req,**kwargs):
|
|
calls.append(req.lmc_file)
|
|
return VerifyResult('PASS',library_flag='GREEN')
|
|
result=context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
|
|
assert result.status!='PASS'
|
|
assert (original.parent/'database/icdb.dat').read_bytes()==b'healthy'
|
|
retained=list(original.parent.parent.glob('Work.before-restore-*'))
|
|
assert len(retained)==1 and (retained[0]/'database/icdb.dat').read_bytes()==b'damaged'
|
|
assert len(calls)==1 and request.lmc_file not in calls
|
|
|
|
|
|
def test_shutdown_cannot_abandon_local_work_swap_or_hold_host_channel(recovery_files,monkeypatch):
|
|
from pathlib import Path
|
|
from threading import Thread
|
|
request,original,source=recovery_files
|
|
context=BatchTaskContext(str(request.lmc_file),native_policy='external')
|
|
entered,release,done=Event(),Event(),Event();calls=[];errors=[]
|
|
rename=Path.rename
|
|
def swap(path,target):
|
|
value=rename(path,target)
|
|
if path==original.parent:
|
|
entered.set();assert release.wait(5)
|
|
return value
|
|
monkeypatch.setattr(Path,'rename',swap)
|
|
def verify(req,**kwargs):
|
|
calls.append(req.lmc_file);return VerifyResult('PASS',library_flag='GREEN')
|
|
def run():
|
|
try:context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
|
|
except Exception as error:errors.append(error)
|
|
finally:done.set()
|
|
thread=Thread(target=run);thread.start()
|
|
try:
|
|
assert entered.wait(3)
|
|
state=context.shutdown()
|
|
assert state['shutdown_handoff'] is False
|
|
assert not done.wait(.1)
|
|
def B():
|
|
with native_channel():return True
|
|
assert BatchTaskContext(str(request.lmc_file.parent.parent/'B/library.lmc'),native_policy='external').run(B)
|
|
finally:
|
|
release.set();thread.join(5)
|
|
assert done.is_set() and len(calls)==1
|
|
assert (original.parent/'database/icdb.dat').read_bytes()==b'healthy'
|
|
|
|
|
|
def test_cancelled_failed_install_rolls_back_before_releasing_commit(recovery_files,monkeypatch):
|
|
from pathlib import Path
|
|
request,original,source=recovery_files;calls=[]
|
|
context=BatchTaskContext(str(request.lmc_file),native_policy='external');rename=Path.rename
|
|
def swap(path,target):
|
|
if path.name.startswith('Work.restore-'):
|
|
context.cancel();raise OSError('install rename refused')
|
|
return rename(path,target)
|
|
monkeypatch.setattr(Path,'rename',swap)
|
|
def verify(req,**kwargs):
|
|
calls.append(req.lmc_file);return VerifyResult('PASS',library_flag='GREEN')
|
|
result=context.run(lambda:recovery.recover_from_backup(request,source,verifier=verify,idle_check=lambda *a:None))
|
|
assert result.status!='PASS' and 'install rename refused' in result.stderr
|
|
assert (original.parent/'database/icdb.dat').read_bytes()==b'damaged'
|
|
assert not list(original.parent.parent.glob('Work.before-restore-*'))
|
|
assert len(calls)==1
|
|
|
|
|
|
def test_metadata_only_context_cannot_enter_local_commit(tmp_path):
|
|
from cbb_refresh_verify.operation_queue import local_commit
|
|
mutations=[]
|
|
def commit():
|
|
with local_commit():mutations.append('rename')
|
|
with pytest.raises(RuntimeError,match='Read-only metadata'):
|
|
BatchTaskContext(str(tmp_path/'lib.lmc')).run(commit,metadata_only=True)
|
|
assert not mutations
|