Ignore queued management completion after its Qt receiver is destroyed. Isolate unrelated WMI and file-proof latency in simulated lifecycle tests while retaining real occupancy and writer-barrier coverage. Validation: 2428 tests passed, compile and whitespace checks passed, scoped review clean. Field ICDB inconsistency and Windows 10/production SMB remain unaccepted.
394 lines
19 KiB
Python
394 lines
19 KiB
Python
"""Real lifecycle regressions from the independent Task 2 review."""
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from threading import Event, Thread
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from test_management import setup as management_setup
|
|
|
|
|
|
def wait_done(context):
|
|
deadline = time.monotonic()+3
|
|
while context.snapshot()['pending'] and time.monotonic()<deadline:
|
|
time.sleep(.01)
|
|
assert not context.snapshot()['pending']
|
|
|
|
|
|
def test_management_result_survives_actual_late_release_read_handoff(management_setup, monkeypatch):
|
|
from cbb_refresh_verify.management import run_management_batch
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
|
|
from cbb_refresh_verify.session_record import write_session_record
|
|
request, adapter, _ = management_setup
|
|
entered, resume = Event(), Event()
|
|
session = []
|
|
remove = adapter.remove
|
|
def fail_remove(req, evidence):
|
|
native = remove(req, evidence)
|
|
path = evidence.parent/'session.json'
|
|
write_session_record(path, dict(operation_id='failed-remove', library=str(req.library),
|
|
block=req.name, processes=[]))
|
|
session.append(path)
|
|
return replace(native, success=False, safe_to_continue=False, detail='exact native removal failure')
|
|
adapter.remove = fail_remove
|
|
opening = Path.open
|
|
def open_path(path, *args, **kwargs):
|
|
if session and path == session[0] and args == ('rb',):
|
|
entered.set()
|
|
assert resume.wait(4)
|
|
return opening(path, *args, **kwargs)
|
|
monkeypatch.setattr(Path, 'open', open_path)
|
|
context = BatchTaskContext(library=str(request.library), targets=(request.name,))
|
|
queue = NativeQueue()
|
|
try:
|
|
results = run_management_batch([request], adapter=adapter, idle_check=lambda *_: None,
|
|
item_executor=lambda req, action: queue.run(action, context=context, timeout_seconds=1))
|
|
assert entered.is_set()
|
|
result = results[0]
|
|
assert result.status == 'RECOVERY_REQUIRED'
|
|
assert result.detail.startswith('exact native removal failure')
|
|
assert result.evidence_path.is_file()
|
|
assert result.change_report['status'] == 'UNCONFIRMED'
|
|
assert result.release_proof['lease_state'] == 'PENDING'
|
|
assert not result.safe_to_continue
|
|
finally:
|
|
resume.set()
|
|
wait_done(context)
|
|
|
|
|
|
@pytest.mark.parametrize('status,expected', [('UPLOAD_FAILED', 'UPLOAD_FAILED'), ('UPLOADED', 'RECOVERY_REQUIRED')])
|
|
def test_management_batch_preserves_carried_handoff_result(management_setup, status, expected):
|
|
from cbb_refresh_verify.management import ManagementResult, run_management_batch
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff
|
|
request, _, _ = management_setup
|
|
known = ManagementResult(request.operation, request.library, request.name, request.partition,
|
|
request.source, request.auto_verify, 'complete', status, False, 'known native detail',
|
|
request.evidence_dir/'checkpoint.json', change_report={'status': 'UNCONFIRMED'})
|
|
context = BatchTaskContext(library=str(request.library))
|
|
def executor(req, action):
|
|
raise TaskHandoff(context, result=known)
|
|
result = run_management_batch([request], item_executor=executor)[0]
|
|
assert result.status == expected and result.detail == known.detail
|
|
assert result.evidence_path == known.evidence_path and result.change_report == known.change_report
|
|
assert result.release_proof['lease_state'] == 'PENDING' and not result.safe_to_continue
|
|
|
|
|
|
def test_exited_verify_worker_outcome_read_is_bounded_and_releases_channel(tmp_path, monkeypatch):
|
|
import json
|
|
from test_verify_runner import fixture, request
|
|
from cbb_refresh_verify.verify_runner import run_verify
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext, native_channel
|
|
fixture(tmp_path)
|
|
entered, resume, completed, stop = Event(), Event(), Event(), Event()
|
|
probes, results, errors = [], [], []
|
|
class ExitedWorker:
|
|
desktop = None
|
|
returncode = 0
|
|
def __init__(self, command, *args, **kwargs):
|
|
root = Path(command[-1]).parent
|
|
(root/'outcome.json').write_text(json.dumps(dict(status='PASS')))
|
|
def poll(self): return 0
|
|
def close(self): pass
|
|
monkeypatch.setattr('cbb_refresh_verify.desktop_worker.DesktopWorker', ExitedWorker)
|
|
# The fake worker has no native clients; cleanup probes are covered independently.
|
|
monkeypatch.setattr('cbb_refresh_verify.verify_runner._cleanup_session_processes',
|
|
lambda *args, **kwargs: {'safe_to_continue': True, 'remaining_pids': []})
|
|
original = Path.read_text
|
|
def read(path, *args, **kwargs):
|
|
if path.name == 'outcome.json':
|
|
entered.set()
|
|
assert resume.wait(4)
|
|
return original(path, *args, **kwargs)
|
|
monkeypatch.setattr(Path, 'read_text', read)
|
|
context = BatchTaskContext(library=str(tmp_path/'lib.lmc'))
|
|
def run():
|
|
try:
|
|
results.append(run_verify(request(tmp_path), stop=stop, context=context,
|
|
admission_timeout_seconds=1, port_check=lambda: (True, ''),
|
|
flag_reader=lambda *a: probes.append('late native status read')))
|
|
except Exception as error:
|
|
errors.append(error)
|
|
finally:
|
|
completed.set()
|
|
thread = Thread(target=run)
|
|
thread.start()
|
|
try:
|
|
assert entered.wait(3), (results, errors)
|
|
stop.set()
|
|
assert completed.wait(1.5), 'exited worker metadata retained native phase'
|
|
with native_channel():
|
|
pass
|
|
assert not errors and not results[0].safe_to_continue
|
|
resume.set()
|
|
wait_done(context)
|
|
assert probes == []
|
|
finally:
|
|
resume.set()
|
|
thread.join(4)
|
|
wait_done(context)
|
|
|
|
|
|
def test_repair_session_write_before_gate_is_bounded_without_host_channel(tmp_path, monkeypatch):
|
|
from cbb_refresh_verify.native_repair import run_native_repair, RepairTarget
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, native_channel
|
|
from cbb_refresh_verify import session_record
|
|
entered, resume, completed = Event(), Event(), Event()
|
|
errors = []
|
|
class GatedWorker:
|
|
returncode = 0
|
|
def __init__(self, command, *args, **kwargs): pass
|
|
def bind_cancellation(self, path): pass
|
|
def bind_session(self, job):
|
|
session_record.write_session_record(Path(job['session_record_path']),
|
|
{'operation_id': job['operation_id']})
|
|
def poll(self): return 0
|
|
def close(self): pass
|
|
monkeypatch.setattr('cbb_refresh_verify.desktop_worker.DesktopWorker', GatedWorker)
|
|
original = Path.open
|
|
def opening(path, *args, **kwargs):
|
|
if path.name.startswith('session.json.') and path.suffix == '.pending':
|
|
entered.set()
|
|
assert resume.wait(4)
|
|
return original(path, *args, **kwargs)
|
|
monkeypatch.setattr(Path, 'open', opening)
|
|
context = BatchTaskContext(library=str(tmp_path/'A.lmc'))
|
|
def run():
|
|
try:
|
|
context.run(lambda: run_native_repair(RepairTarget(tmp_path/'A.prj', tmp_path/'database'),
|
|
tmp_path/'evidence'), timeout_seconds=.1)
|
|
except Exception as error:
|
|
errors.append(error)
|
|
finally:
|
|
completed.set()
|
|
thread = Thread(target=run)
|
|
thread.start()
|
|
try:
|
|
assert entered.wait(2)
|
|
assert completed.wait(.8), 'Repair pre-gate journal retained native phase'
|
|
with native_channel():
|
|
pass
|
|
assert len(errors) == 1 and isinstance(errors[0], TaskHandoff)
|
|
resume.set()
|
|
wait_done(context)
|
|
assert not (tmp_path/'evidence/start-repair.signal').exists()
|
|
finally:
|
|
resume.set()
|
|
thread.join(4)
|
|
wait_done(context)
|
|
|
|
|
|
@pytest.mark.parametrize('control', ['save', 'cancel'])
|
|
def test_handed_off_worker_accepts_exact_controls_but_excludes_new_writers(tmp_path, monkeypatch, control):
|
|
import subprocess
|
|
import sys
|
|
from cbb_refresh_verify.operation_queue import BatchTaskContext, TaskHandoff, native_channel, hold_native_handoff
|
|
from cbb_refresh_verify import residual_sessions as sessions
|
|
from cbb_refresh_verify.session_record import write_session_record
|
|
library = tmp_path/'library/lib.lmc'
|
|
library.parent.mkdir()
|
|
library.touch()
|
|
context = BatchTaskContext(library=str(library))
|
|
child = []
|
|
ready, answered = Event(), Event()
|
|
errors = []
|
|
script = ('from cbb_refresh_verify.desktop_worker import worker_native_channel\n'
|
|
'with worker_native_channel():\n print("ready",flush=True)\n input()\n')
|
|
def action():
|
|
with native_channel():
|
|
process = subprocess.Popen([sys.executable, '-c', script], stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, text=True)
|
|
child.append(process)
|
|
assert process.stdout.readline().strip() == 'ready'
|
|
record = tmp_path/'session.json'
|
|
write_session_record(record, dict(operation_id=context.operation_id, library=str(library)))
|
|
ready.set()
|
|
hold_native_handoff(process, record)
|
|
controller = None
|
|
try:
|
|
with pytest.raises(TaskHandoff):
|
|
context.run(action)
|
|
assert ready.is_set() and child[0].poll() is None
|
|
row = dict(pid=child[0].pid, name='iCDBProjectBackup.exe', created='exact-child',
|
|
executable='C:/Mentor/iCDBProjectBackup.exe', command=f'backup -p "{library.parent}/A.prj"',
|
|
scope_confirmed=True, cancel_prompt={'title': 'captured'})
|
|
def answer(observed, *args, **kwargs):
|
|
assert observed['pid'] == child[0].pid
|
|
child[0].communicate('\n', timeout=3)
|
|
answered.set()
|
|
return {'release_status': 'RELEASED'}
|
|
if control == 'save':
|
|
monkeypatch.setattr(sessions, '_answer_save_question', answer)
|
|
command = lambda: sessions.answer_save_question(row, 6)
|
|
else:
|
|
monkeypatch.setattr(sessions, '_request_native_cancel', answer)
|
|
monkeypatch.setattr(sessions, 'query_process_snapshot', lambda: [row] if child[0].poll() is None else [])
|
|
monkeypatch.setattr(sessions, '_process_windows', lambda _: {})
|
|
# Keep this real writer-barrier/control test independent of file-probe startup latency.
|
|
monkeypatch.setattr(sessions, '_finish_library_release',
|
|
lambda *args, **kwargs: {'status': 'RELEASE_CONFIRMED', 'library': str(library)})
|
|
command = lambda: sessions.release_residual_sessions(dict(library=str(library), sessions=[row]),
|
|
tmp_path/'controls', cancel_native=True)
|
|
def control_call():
|
|
try: command()
|
|
except Exception as error: errors.append(error)
|
|
# New writes still fail, but the exact corrective control can progress.
|
|
with pytest.raises(RuntimeError):
|
|
with native_channel():
|
|
pytest.fail('overlapping new writer admitted')
|
|
controller = Thread(target=control_call)
|
|
controller.start()
|
|
assert answered.wait(1), errors
|
|
controller.join(5)
|
|
assert not controller.is_alive() and not errors
|
|
wait_done(context)
|
|
with native_channel():
|
|
pass
|
|
finally:
|
|
if child and child[0].poll() is None:
|
|
child[0].communicate('\n', timeout=3)
|
|
if controller is not None: controller.join(5)
|
|
wait_done(context)
|
|
|
|
|
|
@pytest.mark.parametrize('mode', ['pending', 'acknowledged', 'assignment-failed', 'cancel-after-registration'])
|
|
def test_real_async_add_unwind_and_parent_exit_keep_only_pending_writer_scope(tmp_path, mode):
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import win32api, win32con, win32event
|
|
from test_management import project
|
|
from cbb_refresh_verify import library_lock as locks
|
|
from cbb_refresh_verify.operation_queue import native_channel
|
|
library = tmp_path/'library/lib.lmc'
|
|
library.parent.mkdir()
|
|
library.touch()
|
|
source = project(tmp_path/'source', name='Block')
|
|
for path in (source, source.parent/'PCB/pcb.prj'):
|
|
with path.open('a', encoding='utf-8') as stream:
|
|
stream.write(f'KEY CentralLibrary "{library}"\n')
|
|
root = tmp_path/'native-add'
|
|
root.mkdir()
|
|
helper = Path(__file__).parent/'support/async_add_lifecycle_fixture.py'
|
|
parent_script = '''import sys,json,time,subprocess,os,win32api,win32con
|
|
from pathlib import Path
|
|
from cbb_refresh_verify.library_lock import LibraryLock,duplicate_worker_lease
|
|
from cbb_refresh_verify.session_record import register_worker_session,process_identity
|
|
library,source,root,helper=map(Path,sys.argv[1:5])
|
|
job=dict(operation='probe_create_add',library=str(library),source=str(source),root=str(root),name='Block',
|
|
operation_id='actual-add',session_record_path=str(root/'session.json'),start_gate=str(root/'gate'),
|
|
cancel_signal=str(root/'cancel'),supervisor_pid=os.getpid(),supervisor_identity=process_identity(os.getpid()),
|
|
timeout_seconds=.3,test_mode=sys.argv[5])
|
|
(root/'request.json').write_text(json.dumps(job))
|
|
with LibraryLock(library,checkpoint=False):
|
|
with (root/'worker-output.txt').open('w') as output:
|
|
worker=subprocess.Popen([sys.executable,str(helper),str(root/'request.json')],stdout=output,stderr=output)
|
|
(root.parent/'worker-pid').write_text(str(worker.pid))
|
|
handle=win32api.OpenProcess(win32con.PROCESS_DUP_HANDLE,False,worker.pid)
|
|
try: inherited=duplicate_worker_lease(library,handle)
|
|
finally: handle.Close()
|
|
register_worker_session(worker,{**job,'worker_lease_handle':inherited})
|
|
(root/'gate').touch()
|
|
deadline=time.monotonic()+12
|
|
while not (root.parent/'add-unwound').exists() and time.monotonic()<deadline: time.sleep(.02)
|
|
assert (root.parent/'add-unwound').exists()
|
|
'''
|
|
parent = subprocess.Popen([sys.executable, '-c', parent_script, str(library), str(source), str(root), str(helper), mode])
|
|
handles = []
|
|
def open_pid_file(name):
|
|
pid = int((tmp_path/name).read_text())
|
|
handle = win32api.OpenProcess(win32con.SYNCHRONIZE, False, pid)
|
|
handles.append(handle)
|
|
return handle
|
|
try:
|
|
code = parent.wait(timeout=15)
|
|
assert code == 0, (root/'worker-output.txt').read_text(errors='replace')
|
|
if mode in ('assignment-failed', 'cancel-after-registration'):
|
|
assert not (tmp_path/'add-dispatched').exists()
|
|
detail = 'exact native job admission denied' if mode == 'assignment-failed' else 'NATIVE_ADD_CANCELLED'
|
|
assert detail in (root/'result.json').read_text()
|
|
with native_channel(): pass
|
|
with locks.LibraryLock(library, checkpoint=False): pass
|
|
return
|
|
assert (tmp_path/'add-dispatched').exists()
|
|
lm = open_pid_file('lm-pid')
|
|
if mode == 'pending':
|
|
worker = open_pid_file('worker-pid')
|
|
from cbb_refresh_verify.session_record import read_session_record
|
|
journal = read_session_record(root/'worker-state.json', evidence_root=root)
|
|
identity = next(p for p in journal['processes'] if p['pid'] == int((tmp_path/'lm-pid').read_text()))
|
|
assert identity['created_handle'] and identity['image'] and identity['owner_sid']
|
|
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
|
with native_channel(): pass
|
|
with pytest.raises(locks.LibraryBusy):
|
|
with locks.LibraryLock(library, checkpoint=False): pass
|
|
(tmp_path/'lm-exit').touch()
|
|
assert win32event.WaitForSingleObject(lm, 3000) == win32con.WAIT_OBJECT_0
|
|
assert win32event.WaitForSingleObject(worker, 0) != win32con.WAIT_OBJECT_0
|
|
deadline = time.monotonic()+2
|
|
descendant = int((tmp_path/'descendant-pid').read_text())
|
|
while time.monotonic()<deadline:
|
|
journal = read_session_record(root/'worker-state.json', evidence_root=root)
|
|
if any(p['pid'] == descendant for p in journal['processes']): break
|
|
time.sleep(.02)
|
|
assert any(p['pid'] == descendant and p['created_handle'] for p in journal['processes'])
|
|
with pytest.raises(RuntimeError, match='NATIVE_WRITER_ACTIVE'):
|
|
with native_channel(): pass
|
|
with pytest.raises(locks.LibraryBusy):
|
|
with locks.LibraryLock(library, checkpoint=False): pass
|
|
(tmp_path/'native-done').touch()
|
|
assert win32event.WaitForSingleObject(worker, 5000) == win32con.WAIT_OBJECT_0
|
|
else:
|
|
deadline = time.monotonic()+3
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
with native_channel(): pass
|
|
break
|
|
except RuntimeError:
|
|
time.sleep(.02)
|
|
with native_channel(): pass
|
|
assert win32event.WaitForSingleObject(lm, 0) != win32con.WAIT_OBJECT_0, 'idle LM was killed'
|
|
with locks.LibraryLock(library, checkpoint=False): pass
|
|
finally:
|
|
(tmp_path/'lm-exit').touch()
|
|
(tmp_path/'native-done').touch()
|
|
for handle in handles:
|
|
win32event.WaitForSingleObject(handle, 5000)
|
|
handle.Close()
|
|
if (tmp_path/'worker-pid').exists():
|
|
try:
|
|
worker = open_pid_file('worker-pid')
|
|
except Exception as error:
|
|
if getattr(error, 'winerror', None) != 87: raise
|
|
else:
|
|
assert win32event.WaitForSingleObject(worker, 5000) == win32con.WAIT_OBJECT_0
|
|
worker.Close()
|
|
parent.wait(timeout=15)
|
|
|
|
|
|
def test_add_descendant_snapshot_cannot_journal_an_unrelated_reused_pid(tmp_path, monkeypatch):
|
|
import subprocess
|
|
import sys
|
|
from cbb_refresh_verify import desktop_worker
|
|
from cbb_refresh_verify.session_record import write_session_record, read_session_record
|
|
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
|
owner = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
|
|
foreign = subprocess.Popen([sys.executable, '-c', 'input()'], stdin=subprocess.PIPE, text=True)
|
|
operation = None
|
|
query = desktop_worker.win32job.QueryInformationJobObject
|
|
try:
|
|
job = dict(operation_id='descendant-identity', session_record_path=str(tmp_path/'session.json'))
|
|
write_session_record(tmp_path/'worker-state.json', dict(operation_id=job['operation_id'], processes=[]))
|
|
operation = desktop_worker.register_pending_native_operation(job, {owner.pid})
|
|
monkeypatch.setattr(desktop_worker.win32job, 'QueryInformationJobObject',
|
|
lambda *args: [owner.pid, foreign.pid])
|
|
assert operation.poll() is None
|
|
journal = read_session_record(tmp_path/'worker-state.json', evidence_root=tmp_path)
|
|
assert not any(p['pid'] == foreign.pid for p in journal['processes'])
|
|
finally:
|
|
owner.communicate('\n', timeout=3)
|
|
foreign.communicate('\n', timeout=3)
|
|
monkeypatch.setattr(desktop_worker.win32job, 'QueryInformationJobObject', query)
|
|
if operation is not None:
|
|
assert operation.poll() == 0
|