diff --git a/src/cbb_refresh_verify/session_record.py b/src/cbb_refresh_verify/session_record.py new file mode 100644 index 0000000..fafdf46 --- /dev/null +++ b/src/cbb_refresh_verify/session_record.py @@ -0,0 +1,312 @@ +"""Bounded, observation-only journal for native calls that outlive a task.""" +from contextlib import ExitStack +from datetime import datetime, timezone +import json +import logging +import os +from pathlib import Path +import stat +import threading +from uuid import uuid4 + +PROTOCOL = 'cbb-runtime-session' +MAX_BYTES = 64 * 1024 +IDENTITY_KEYS = ('pid', 'created_handle', 'image', 'owner_sid', 'session_id') +_observers = {} +_observer_guard = threading.Lock() +_retained_libraries = {} + + +def _library_key(library): + return os.path.normcase(os.path.normpath(str(library))) + + +def library_available(library): + """Cached queue predicate: no file, process or network queries.""" + key = _library_key(library) + with _observer_guard: + return not any(key in aliases for aliases in _retained_libraries.values()) + + +def register_worker_session(worker, job): + """The launcher must persist identity before opening the worker's gate.""" + path = Path(job['session_record_path']) + write_session_record(path, {k: job.get(k) for k in ( + 'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal')} + | {'processes': [{'role': 'worker', **process_identity(worker.pid)}]}) + worker.session_record_path = path + worker.cbb_library = job['library'] + + +def update_worker_state(job, phase, *, identities=(), write_outcome=None): + if not job.get('session_record_path'): + return + path = Path(job['session_record_path']).with_name('worker-state.json') + try: + state = ({'operation_id': job['operation_id'], 'processes': []} if not path.exists() + else read_session_record(path, evidence_root=path.parent)) + if state['operation_id'] != job['operation_id']: + raise ValueError('工作状态操作身份不一致') + for role, pid in identities: + if pid and not any(p['pid'] == pid for p in state['processes']): + state['processes'].append({'role': role, **process_identity(pid)}) + state['phase'] = phase + if write_outcome is not None: + state['write_outcome'] = write_outcome + write_session_record(path, state) + except Exception: + # An unavailable journal cannot interrupt an already executing COM call. + logging.getLogger(__name__).warning('Worker session journal unavailable', exc_info=True) + + +def process_identity(pid): + from .residual_sessions import _process_handle, _handle_identity, _force_ownership + with _process_handle(pid) as handle: + identity = _handle_identity(handle) + identity.update(_force_ownership(handle, pid)) + return {'pid': int(pid), **{k: identity[k] for k in IDENTITY_KEYS if k != 'pid'}} + + +def write_session_record(path, record): + path = Path(path) + value = {'protocol': PROTOCOL, 'version': 1, **record, + 'recorded_at': datetime.now(timezone.utc).isoformat()} + data = json.dumps(value, ensure_ascii=False, default=str, indent=2).encode('utf-8') + if len(data) > MAX_BYTES: + raise ValueError('会话记录超过大小限制') + pending = path.with_name(path.name + '.' + uuid4().hex + '.pending') + try: + with pending.open('xb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(pending, path) + finally: + pending.unlink(missing_ok=True) + + +def read_session_record(path, *, evidence_root): + path, root = Path(path).absolute(), Path(evidence_root).absolute() + if not path.is_relative_to(root) or path == root: + raise ValueError('会话记录位于受控证据目录之外') + # Check every component before resolve, which would hide a junction. + for item in (path, *path.parents): + attrs = item.lstat() + if getattr(attrs, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT: + raise ValueError('会话记录路径包含重解析点') + with path.open('rb') as stream: + data = stream.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError('会话记录超过大小限制') + value = json.loads(data) + if (not isinstance(value, dict) or value.get('protocol') != PROTOCOL or + type(value.get('version')) is not int or value['version'] != 1 or + not isinstance(value.get('operation_id'), str) or not value['operation_id']): + raise ValueError('会话记录协议无效') + return value + + +def _session(path, root): + value = read_session_record(path, evidence_root=root) + if (not isinstance(value.get('library'), str) or not Path(value['library']).is_absolute() + or Path(value['library']).suffix.casefold() != '.lmc'): + raise ValueError('会话库身份无效') + state_path = Path(path).with_name('worker-state.json') + state = {} + if state_path.exists(): + state = read_session_record(state_path, evidence_root=root) + if state['operation_id'] != value['operation_id']: + raise ValueError('工作进程状态不属于本次操作') + processes = value.get('processes', []) + state.get('processes', []) + if not processes: + raise ValueError('会话缺少精确进程身份') + for process in processes: + if (not isinstance(process, dict) or any(k not in process for k in IDENTITY_KEYS) + or type(process['pid']) is not int or process['pid'] <= 0): + raise ValueError('会话进程身份不完整') + return value, state, processes + + +def _result(status, detail='', *, session=None, state=None, proof=None): + return {'status': status, 'detail': detail, 'library': (session or {}).get('library'), + 'operation_id': (session or {}).get('operation_id'), + 'phase': (state or {}).get('phase', 'UNCONFIRMED'), 'release_proof': proof} + + +def _reconcile(path, root, handles, stack): + import pywintypes, win32api, win32con, win32event + from .residual_sessions import _handle_identity, _force_ownership + session, state, processes = _session(path, root) + active = False + for identity in processes: + if identity.get('role') == 'supervisor': + continue + key = tuple(identity[k] for k in IDENTITY_KEYS) + if key not in handles: + try: + handle = win32api.OpenProcess(win32con.SYNCHRONIZE | 0x1000, False, identity['pid']) + except pywintypes.error as error: + if error.winerror != 87: # Access denied is not an exit observation. + raise + handles[key] = None + else: + stack.callback(handle.Close) + if win32event.WaitForSingleObject(handle, 0) == win32con.WAIT_OBJECT_0: + handles[key] = handle + continue + actual = {'pid': identity['pid'], **_handle_identity(handle), + **_force_ownership(handle, identity['pid'])} + if any(actual[k] != identity[k] for k in IDENTITY_KEYS): + return _result('IDENTITY_CHANGED', '记录中的进程身份已变化,仅供观察', + session=session, state=state) + handles[key] = handle + handle = handles[key] + if handle is not None and win32event.WaitForSingleObject(handle, 0) != win32con.WAIT_OBJECT_0: + active = True + if active: + return _result('ACTIVE', '原生调用尚未确认返回', session=session, state=state) + from .library_lock import release_retained_library + try: + proof = release_retained_library(session['library'], Path(path).parent/'release-proof', + block_name=session.get('block')) + except Exception as error: + return _result('UNCONFIRMED', str(error), session=session, state=state) + if proof.get('status') != 'RELEASE_CONFIRMED': + return _result('UNCONFIRMED', '缺少独立资源释放证明', session=session, state=state) + from .network_path import release_retained_path_leases + release_retained_path_leases(proof) + return _result('RELEASED', session=session, state=state, proof=proof) + + +def reconcile_session_record(path, *, evidence_root): + with ExitStack() as stack: + try: + return _reconcile(Path(path), Path(evidence_root), {}, stack) + except Exception as error: + return _result('UNCONFIRMED', str(error)) + + +def observe_retained_session(path, *, evidence_root, stop=None, progress=None, process=None): + stop = stop if stop is not None else threading.Event() + observation = Path(path).with_name('observation-' + uuid4().hex + '.json') + with ExitStack() as stack: + handles = {} + while True: + try: + result = _reconcile(Path(path), Path(evidence_root), handles, stack) + except Exception as error: + result = _result('UNCONFIRMED', str(error)) + if process is not None and process.poll() is not None and getattr(process, 'cbb_library', None): + try: + from .library_lock import release_retained_library + proof = release_retained_library(process.cbb_library, Path(path).parent/'release-proof') + if proof.get('status') == 'RELEASE_CONFIRMED': + result = _result('RELEASED', '会话记录不可读;由原进程句柄及独立资源证明确认释放', + session={'library': str(process.cbb_library)}, proof=proof) + except Exception: + pass + try: + write_session_record(observation, {**result, 'operation_id': result['operation_id'] or 'unknown'}) + except (OSError, ValueError): + logging.getLogger(__name__).warning('Observation journal unavailable', exc_info=True) + if progress is not None: + try: + progress(result) + except Exception: + logging.getLogger(__name__).warning('Observation callback failed', exc_info=True) + if result['status'] in ('RELEASED', 'IDENTITY_CHANGED') or stop.is_set(): + return result + if not handles and process is not None and process.poll() is None: + stop.wait(1) + continue # The exact owned handle survives a failed journal. + if not handles and process is None: + return result + stop.wait(5 if result['status'] == 'UNCONFIRMED' else 1) + + +def handoff_session(path, *, evidence_root, process=None, progress=None, stop=None): + """Retain exact handles in memory until the observer independently releases.""" + key = str(Path(path).absolute()).casefold() + try: + session = read_session_record(path, evidence_root=evidence_root) + library = session['library'] + except (OSError, ValueError, KeyError): + library = getattr(process, 'cbb_library', None) + aliases = {_library_key(library), _library_key(Path(library).resolve())} if library else set() + with _observer_guard: + if key in _observers: + return + _retained_libraries[key] = aliases + def observe(): + result = None + try: + result = observe_retained_session(path, evidence_root=evidence_root, progress=progress, + stop=stop, process=process) + if process is not None and process.poll() is not None: + process.close() + return result + finally: + with _observer_guard: + _observers.pop(key, None) + if result and result['status'] == 'RELEASED': + _retained_libraries.pop(key, None) + thread = threading.Thread(target=observe, name='retained-native-session', daemon=True) + _observers[key] = (thread, process) + thread.start() + + +def discover_sessions(evidence_root, *, library=None): + """Only our bounded journals are inputs; never execute a stored command.""" + root = Path(evidence_root) + found = [] + if not root.is_dir(): + return found + visited = 0 + for folder, directories, files in os.walk(root, followlinks=False): + directories[:] = [d for d in directories if not ( + getattr((Path(folder)/d).lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT)] + visited += 1 + if visited > 10000: + break + if 'session.json' not in files: + continue + path = Path(folder)/'session.json' + try: + value = read_session_record(path, evidence_root=root) + completed = False + for name in files: + if not (name.startswith('observation-') and name.endswith('.json')): + continue + observation = read_session_record(Path(folder)/name, evidence_root=root) + if observation['operation_id'] == value['operation_id'] and ( + observation.get('status') == 'RELEASED' and observation.get('release_proof') + or observation.get('status') == 'FINISHED' + and observation.get('native_release_confirmed') is True + and observation.get('worker_exited') is True): + completed = True + break + if completed: + continue + if library is not None and _library_key(value.get('library', '')) != _library_key(library): + continue + value = {**value, 'path': str(path), **reconcile_session_record(path, evidence_root=root)} + except (OSError, ValueError): + continue + found.append(value) + return found + + +def active_write_records(library): + """Called by the background residual inspector, never by a queue predicate.""" + with _observer_guard: + paths = [Path(key) for key, aliases in _retained_libraries.items() + if _library_key(library) in aliases] + result = [] + for path in paths: + try: + session, state, _ = _session(path, path.parent) + if state.get('phase') in ('NATIVE_CALL', 'CLOSING', 'UNCONFIRMED'): + result.append({**session, 'phase': state['phase']}) + except (OSError, ValueError): + result.append({'phase': 'UNCONFIRMED'}) + return result diff --git a/tests/test_session_record.py b/tests/test_session_record.py new file mode 100644 index 0000000..face840 --- /dev/null +++ b/tests/test_session_record.py @@ -0,0 +1,145 @@ +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from cbb_refresh_verify import session_record as sessions + + +def record(tmp_path, processes): + path = tmp_path/'session.json' + sessions.write_session_record(path, dict(operation_id='test-session', operation='verify', + library=str(tmp_path/'lib.lmc'), block='A', processes=processes)) + return path + + +@pytest.fixture +def child(): + process = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']) + try: + yield process + finally: + process.terminate() + process.wait(5) + + +def test_round_trip_and_exact_live_process(tmp_path, child): + identity = sessions.process_identity(child.pid) + path = record(tmp_path, [dict(role='worker', **identity)]) + value = sessions.read_session_record(path, evidence_root=tmp_path) + assert value['protocol'] == 'cbb-runtime-session' + assert value['version'] == 1 + result = sessions.reconcile_session_record(path, evidence_root=tmp_path) + assert result['status'] == 'ACTIVE' + assert result['release_proof'] is None + assert child.poll() is None + + +@pytest.mark.parametrize('key,value', [('created_handle', 'old'), ('image', 'C:/old.exe'), + ('owner_sid', 'S-1-old'), ('session_id', -1)]) +def test_reused_pid_is_observed_never_acted_on(tmp_path, child, key, value): + identity = sessions.process_identity(child.pid) + identity[key] = value + path = record(tmp_path, [dict(role='worker', **identity)]) + result = sessions.reconcile_session_record(path, evidence_root=tmp_path) + assert result['status'] == 'IDENTITY_CHANGED' + assert result['release_proof'] is None + assert child.poll() is None + + +@pytest.mark.parametrize('kind', ['outside', 'large', 'invalid', 'unknown_protocol']) +def test_untrusted_record_is_unconfirmed_not_an_occupancy_fact(tmp_path, kind): + owned = tmp_path/'owned' + owned.mkdir() + path = (tmp_path if kind == 'outside' else owned)/'session.json' + path.write_text('x'*65537 if kind == 'large' else '{' if kind == 'invalid' else '{}') + result = sessions.reconcile_session_record(path, evidence_root=owned) + assert result['status'] == 'UNCONFIRMED' + assert result['release_proof'] is None + + +def test_exited_process_alone_is_not_release_proof(tmp_path, child, monkeypatch): + path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))]) + child.terminate() + child.wait(5) + def busy(*a, **kw): + raise OSError('database still occupied') + monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', busy) + result = sessions.reconcile_session_record(path, evidence_root=tmp_path) + assert result['status'] == 'UNCONFIRMED' + assert result['release_proof'] is None + assert 'occupied' in result['detail'] + + +def test_release_uses_independent_proof_and_does_not_replay(tmp_path, child, monkeypatch): + path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))]) + child.terminate() + child.wait(5) + proof = {'status': 'RELEASE_CONFIRMED', 'library': str(tmp_path/'lib.lmc')} + monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', lambda *a, **kw: proof) + result = sessions.reconcile_session_record(path, evidence_root=tmp_path) + assert result['status'] == 'RELEASED' + assert result['release_proof'] == proof + + +def test_observation_can_stop_without_claiming_native_release(tmp_path, child): + import threading + path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))]) + stop = threading.Event() + stop.set() + result = sessions.observe_retained_session(path, evidence_root=tmp_path, stop=stop) + assert result['status'] == 'ACTIVE' + assert result['release_proof'] is None + assert child.poll() is None + + +def test_worker_state_cannot_replace_another_operation(tmp_path, child): + path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))]) + sessions.write_session_record(tmp_path/'worker-state.json', { + 'operation_id': 'different', 'processes': [], 'phase': 'FINISHED'}) + result = sessions.reconcile_session_record(path, evidence_root=tmp_path) + assert result['status'] == 'UNCONFIRMED' + assert child.poll() is None + + +def test_retained_session_yields_global_queue_but_keeps_same_library_waiting(tmp_path, child): + import threading + from cbb_refresh_verify.operation_queue import NativeQueue + from cbb_refresh_verify.session_record import handoff_session, library_available + library = tmp_path/'lib.lmc' + path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))]) + queue = NativeQueue() + observation_stop = threading.Event() + queue.run(lambda: handoff_session(path, evidence_root=tmp_path, stop=observation_stop)) + assert library_available(library) is False + assert library_available(tmp_path/'other.lmc') is True + assert queue.run(lambda: 'other-completed', ready=lambda: library_available(tmp_path/'other.lmc')) == 'other-completed' + stop = threading.Event() + stop.set() + from cbb_refresh_verify.operation_queue import QueueCancelled + with pytest.raises(QueueCancelled): + queue.run(lambda: pytest.fail('overlapping writer'), stop=stop, + ready=lambda: library_available(library)) + observation_stop.set() + + +def test_bad_journal_does_not_drop_in_memory_supervision(tmp_path, child): + import threading + import time + from types import SimpleNamespace + path = tmp_path/'session.json' + path.write_text('{') + stop = threading.Event() + process = SimpleNamespace(poll=child.poll, close=lambda: None, cbb_library=tmp_path/'lib.lmc') + sessions.handoff_session(path, evidence_root=tmp_path, process=process, stop=stop) + key = str(path.absolute()).casefold() + try: + time.sleep(.1) + with sessions._observer_guard: + assert key in sessions._observers, 'active owned handle lost its observer because JSON was unreadable' + assert child.poll() is None + finally: + stop.set()