feat: persist and reconcile owned native sessions
This commit is contained in:
1 parent
7bd7cd3966
commit
97a3de20c9
2 files changed
+457
No files matched your search
@@ -0,0 +1,312 @@
|
||||
"""Bounded, observation-only journal for native calls that outlive a task."""
|
||||
from contextlib import ExitStack
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
PROTOCOL = 'cbb-runtime-session'
|
||||
MAX_BYTES = 64 * 1024
|
||||
IDENTITY_KEYS = ('pid', 'created_handle', 'image', 'owner_sid', 'session_id')
|
||||
_observers = {}
|
||||
_observer_guard = threading.Lock()
|
||||
_retained_libraries = {}
|
||||
|
||||
|
||||
def _library_key(library):
|
||||
return os.path.normcase(os.path.normpath(str(library)))
|
||||
|
||||
|
||||
def library_available(library):
|
||||
"""Cached queue predicate: no file, process or network queries."""
|
||||
key = _library_key(library)
|
||||
with _observer_guard:
|
||||
return not any(key in aliases for aliases in _retained_libraries.values())
|
||||
|
||||
|
||||
def register_worker_session(worker, job):
|
||||
"""The launcher must persist identity before opening the worker's gate."""
|
||||
path = Path(job['session_record_path'])
|
||||
write_session_record(path, {k: job.get(k) for k in (
|
||||
'operation_id', 'operation', 'library', 'block', 'desktop', 'cancel_signal')}
|
||||
| {'processes': [{'role': 'worker', **process_identity(worker.pid)}]})
|
||||
worker.session_record_path = path
|
||||
worker.cbb_library = job['library']
|
||||
|
||||
|
||||
def update_worker_state(job, phase, *, identities=(), write_outcome=None):
|
||||
if not job.get('session_record_path'):
|
||||
return
|
||||
path = Path(job['session_record_path']).with_name('worker-state.json')
|
||||
try:
|
||||
state = ({'operation_id': job['operation_id'], 'processes': []} if not path.exists()
|
||||
else read_session_record(path, evidence_root=path.parent))
|
||||
if state['operation_id'] != job['operation_id']:
|
||||
raise ValueError('工作状态操作身份不一致')
|
||||
for role, pid in identities:
|
||||
if pid and not any(p['pid'] == pid for p in state['processes']):
|
||||
state['processes'].append({'role': role, **process_identity(pid)})
|
||||
state['phase'] = phase
|
||||
if write_outcome is not None:
|
||||
state['write_outcome'] = write_outcome
|
||||
write_session_record(path, state)
|
||||
except Exception:
|
||||
# An unavailable journal cannot interrupt an already executing COM call.
|
||||
logging.getLogger(__name__).warning('Worker session journal unavailable', exc_info=True)
|
||||
|
||||
|
||||
def process_identity(pid):
|
||||
from .residual_sessions import _process_handle, _handle_identity, _force_ownership
|
||||
with _process_handle(pid) as handle:
|
||||
identity = _handle_identity(handle)
|
||||
identity.update(_force_ownership(handle, pid))
|
||||
return {'pid': int(pid), **{k: identity[k] for k in IDENTITY_KEYS if k != 'pid'}}
|
||||
|
||||
|
||||
def write_session_record(path, record):
|
||||
path = Path(path)
|
||||
value = {'protocol': PROTOCOL, 'version': 1, **record,
|
||||
'recorded_at': datetime.now(timezone.utc).isoformat()}
|
||||
data = json.dumps(value, ensure_ascii=False, default=str, indent=2).encode('utf-8')
|
||||
if len(data) > MAX_BYTES:
|
||||
raise ValueError('会话记录超过大小限制')
|
||||
pending = path.with_name(path.name + '.' + uuid4().hex + '.pending')
|
||||
try:
|
||||
with pending.open('xb') as stream:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(pending, path)
|
||||
finally:
|
||||
pending.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def read_session_record(path, *, evidence_root):
|
||||
path, root = Path(path).absolute(), Path(evidence_root).absolute()
|
||||
if not path.is_relative_to(root) or path == root:
|
||||
raise ValueError('会话记录位于受控证据目录之外')
|
||||
# Check every component before resolve, which would hide a junction.
|
||||
for item in (path, *path.parents):
|
||||
attrs = item.lstat()
|
||||
if getattr(attrs, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
raise ValueError('会话记录路径包含重解析点')
|
||||
with path.open('rb') as stream:
|
||||
data = stream.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES:
|
||||
raise ValueError('会话记录超过大小限制')
|
||||
value = json.loads(data)
|
||||
if (not isinstance(value, dict) or value.get('protocol') != PROTOCOL or
|
||||
type(value.get('version')) is not int or value['version'] != 1 or
|
||||
not isinstance(value.get('operation_id'), str) or not value['operation_id']):
|
||||
raise ValueError('会话记录协议无效')
|
||||
return value
|
||||
|
||||
|
||||
def _session(path, root):
|
||||
value = read_session_record(path, evidence_root=root)
|
||||
if (not isinstance(value.get('library'), str) or not Path(value['library']).is_absolute()
|
||||
or Path(value['library']).suffix.casefold() != '.lmc'):
|
||||
raise ValueError('会话库身份无效')
|
||||
state_path = Path(path).with_name('worker-state.json')
|
||||
state = {}
|
||||
if state_path.exists():
|
||||
state = read_session_record(state_path, evidence_root=root)
|
||||
if state['operation_id'] != value['operation_id']:
|
||||
raise ValueError('工作进程状态不属于本次操作')
|
||||
processes = value.get('processes', []) + state.get('processes', [])
|
||||
if not processes:
|
||||
raise ValueError('会话缺少精确进程身份')
|
||||
for process in processes:
|
||||
if (not isinstance(process, dict) or any(k not in process for k in IDENTITY_KEYS)
|
||||
or type(process['pid']) is not int or process['pid'] <= 0):
|
||||
raise ValueError('会话进程身份不完整')
|
||||
return value, state, processes
|
||||
|
||||
|
||||
def _result(status, detail='', *, session=None, state=None, proof=None):
|
||||
return {'status': status, 'detail': detail, 'library': (session or {}).get('library'),
|
||||
'operation_id': (session or {}).get('operation_id'),
|
||||
'phase': (state or {}).get('phase', 'UNCONFIRMED'), 'release_proof': proof}
|
||||
|
||||
|
||||
def _reconcile(path, root, handles, stack):
|
||||
import pywintypes, win32api, win32con, win32event
|
||||
from .residual_sessions import _handle_identity, _force_ownership
|
||||
session, state, processes = _session(path, root)
|
||||
active = False
|
||||
for identity in processes:
|
||||
if identity.get('role') == 'supervisor':
|
||||
continue
|
||||
key = tuple(identity[k] for k in IDENTITY_KEYS)
|
||||
if key not in handles:
|
||||
try:
|
||||
handle = win32api.OpenProcess(win32con.SYNCHRONIZE | 0x1000, False, identity['pid'])
|
||||
except pywintypes.error as error:
|
||||
if error.winerror != 87: # Access denied is not an exit observation.
|
||||
raise
|
||||
handles[key] = None
|
||||
else:
|
||||
stack.callback(handle.Close)
|
||||
if win32event.WaitForSingleObject(handle, 0) == win32con.WAIT_OBJECT_0:
|
||||
handles[key] = handle
|
||||
continue
|
||||
actual = {'pid': identity['pid'], **_handle_identity(handle),
|
||||
**_force_ownership(handle, identity['pid'])}
|
||||
if any(actual[k] != identity[k] for k in IDENTITY_KEYS):
|
||||
return _result('IDENTITY_CHANGED', '记录中的进程身份已变化,仅供观察',
|
||||
session=session, state=state)
|
||||
handles[key] = handle
|
||||
handle = handles[key]
|
||||
if handle is not None and win32event.WaitForSingleObject(handle, 0) != win32con.WAIT_OBJECT_0:
|
||||
active = True
|
||||
if active:
|
||||
return _result('ACTIVE', '原生调用尚未确认返回', session=session, state=state)
|
||||
from .library_lock import release_retained_library
|
||||
try:
|
||||
proof = release_retained_library(session['library'], Path(path).parent/'release-proof',
|
||||
block_name=session.get('block'))
|
||||
except Exception as error:
|
||||
return _result('UNCONFIRMED', str(error), session=session, state=state)
|
||||
if proof.get('status') != 'RELEASE_CONFIRMED':
|
||||
return _result('UNCONFIRMED', '缺少独立资源释放证明', session=session, state=state)
|
||||
from .network_path import release_retained_path_leases
|
||||
release_retained_path_leases(proof)
|
||||
return _result('RELEASED', session=session, state=state, proof=proof)
|
||||
|
||||
|
||||
def reconcile_session_record(path, *, evidence_root):
|
||||
with ExitStack() as stack:
|
||||
try:
|
||||
return _reconcile(Path(path), Path(evidence_root), {}, stack)
|
||||
except Exception as error:
|
||||
return _result('UNCONFIRMED', str(error))
|
||||
|
||||
|
||||
def observe_retained_session(path, *, evidence_root, stop=None, progress=None, process=None):
|
||||
stop = stop if stop is not None else threading.Event()
|
||||
observation = Path(path).with_name('observation-' + uuid4().hex + '.json')
|
||||
with ExitStack() as stack:
|
||||
handles = {}
|
||||
while True:
|
||||
try:
|
||||
result = _reconcile(Path(path), Path(evidence_root), handles, stack)
|
||||
except Exception as error:
|
||||
result = _result('UNCONFIRMED', str(error))
|
||||
if process is not None and process.poll() is not None and getattr(process, 'cbb_library', None):
|
||||
try:
|
||||
from .library_lock import release_retained_library
|
||||
proof = release_retained_library(process.cbb_library, Path(path).parent/'release-proof')
|
||||
if proof.get('status') == 'RELEASE_CONFIRMED':
|
||||
result = _result('RELEASED', '会话记录不可读;由原进程句柄及独立资源证明确认释放',
|
||||
session={'library': str(process.cbb_library)}, proof=proof)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
write_session_record(observation, {**result, 'operation_id': result['operation_id'] or 'unknown'})
|
||||
except (OSError, ValueError):
|
||||
logging.getLogger(__name__).warning('Observation journal unavailable', exc_info=True)
|
||||
if progress is not None:
|
||||
try:
|
||||
progress(result)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning('Observation callback failed', exc_info=True)
|
||||
if result['status'] in ('RELEASED', 'IDENTITY_CHANGED') or stop.is_set():
|
||||
return result
|
||||
if not handles and process is not None and process.poll() is None:
|
||||
stop.wait(1)
|
||||
continue # The exact owned handle survives a failed journal.
|
||||
if not handles and process is None:
|
||||
return result
|
||||
stop.wait(5 if result['status'] == 'UNCONFIRMED' else 1)
|
||||
|
||||
|
||||
def handoff_session(path, *, evidence_root, process=None, progress=None, stop=None):
|
||||
"""Retain exact handles in memory until the observer independently releases."""
|
||||
key = str(Path(path).absolute()).casefold()
|
||||
try:
|
||||
session = read_session_record(path, evidence_root=evidence_root)
|
||||
library = session['library']
|
||||
except (OSError, ValueError, KeyError):
|
||||
library = getattr(process, 'cbb_library', None)
|
||||
aliases = {_library_key(library), _library_key(Path(library).resolve())} if library else set()
|
||||
with _observer_guard:
|
||||
if key in _observers:
|
||||
return
|
||||
_retained_libraries[key] = aliases
|
||||
def observe():
|
||||
result = None
|
||||
try:
|
||||
result = observe_retained_session(path, evidence_root=evidence_root, progress=progress,
|
||||
stop=stop, process=process)
|
||||
if process is not None and process.poll() is not None:
|
||||
process.close()
|
||||
return result
|
||||
finally:
|
||||
with _observer_guard:
|
||||
_observers.pop(key, None)
|
||||
if result and result['status'] == 'RELEASED':
|
||||
_retained_libraries.pop(key, None)
|
||||
thread = threading.Thread(target=observe, name='retained-native-session', daemon=True)
|
||||
_observers[key] = (thread, process)
|
||||
thread.start()
|
||||
|
||||
|
||||
def discover_sessions(evidence_root, *, library=None):
|
||||
"""Only our bounded journals are inputs; never execute a stored command."""
|
||||
root = Path(evidence_root)
|
||||
found = []
|
||||
if not root.is_dir():
|
||||
return found
|
||||
visited = 0
|
||||
for folder, directories, files in os.walk(root, followlinks=False):
|
||||
directories[:] = [d for d in directories if not (
|
||||
getattr((Path(folder)/d).lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT)]
|
||||
visited += 1
|
||||
if visited > 10000:
|
||||
break
|
||||
if 'session.json' not in files:
|
||||
continue
|
||||
path = Path(folder)/'session.json'
|
||||
try:
|
||||
value = read_session_record(path, evidence_root=root)
|
||||
completed = False
|
||||
for name in files:
|
||||
if not (name.startswith('observation-') and name.endswith('.json')):
|
||||
continue
|
||||
observation = read_session_record(Path(folder)/name, evidence_root=root)
|
||||
if observation['operation_id'] == value['operation_id'] and (
|
||||
observation.get('status') == 'RELEASED' and observation.get('release_proof')
|
||||
or observation.get('status') == 'FINISHED'
|
||||
and observation.get('native_release_confirmed') is True
|
||||
and observation.get('worker_exited') is True):
|
||||
completed = True
|
||||
break
|
||||
if completed:
|
||||
continue
|
||||
if library is not None and _library_key(value.get('library', '')) != _library_key(library):
|
||||
continue
|
||||
value = {**value, 'path': str(path), **reconcile_session_record(path, evidence_root=root)}
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
found.append(value)
|
||||
return found
|
||||
|
||||
|
||||
def active_write_records(library):
|
||||
"""Called by the background residual inspector, never by a queue predicate."""
|
||||
with _observer_guard:
|
||||
paths = [Path(key) for key, aliases in _retained_libraries.items()
|
||||
if _library_key(library) in aliases]
|
||||
result = []
|
||||
for path in paths:
|
||||
try:
|
||||
session, state, _ = _session(path, path.parent)
|
||||
if state.get('phase') in ('NATIVE_CALL', 'CLOSING', 'UNCONFIRMED'):
|
||||
result.append({**session, 'phase': state['phase']})
|
||||
except (OSError, ValueError):
|
||||
result.append({'phase': 'UNCONFIRMED'})
|
||||
return result
|
||||
@@ -0,0 +1,145 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from cbb_refresh_verify import session_record as sessions
|
||||
|
||||
|
||||
def record(tmp_path, processes):
|
||||
path = tmp_path/'session.json'
|
||||
sessions.write_session_record(path, dict(operation_id='test-session', operation='verify',
|
||||
library=str(tmp_path/'lib.lmc'), block='A', processes=processes))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def child():
|
||||
process = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
|
||||
try:
|
||||
yield process
|
||||
finally:
|
||||
process.terminate()
|
||||
process.wait(5)
|
||||
|
||||
|
||||
def test_round_trip_and_exact_live_process(tmp_path, child):
|
||||
identity = sessions.process_identity(child.pid)
|
||||
path = record(tmp_path, [dict(role='worker', **identity)])
|
||||
value = sessions.read_session_record(path, evidence_root=tmp_path)
|
||||
assert value['protocol'] == 'cbb-runtime-session'
|
||||
assert value['version'] == 1
|
||||
result = sessions.reconcile_session_record(path, evidence_root=tmp_path)
|
||||
assert result['status'] == 'ACTIVE'
|
||||
assert result['release_proof'] is None
|
||||
assert child.poll() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('key,value', [('created_handle', 'old'), ('image', 'C:/old.exe'),
|
||||
('owner_sid', 'S-1-old'), ('session_id', -1)])
|
||||
def test_reused_pid_is_observed_never_acted_on(tmp_path, child, key, value):
|
||||
identity = sessions.process_identity(child.pid)
|
||||
identity[key] = value
|
||||
path = record(tmp_path, [dict(role='worker', **identity)])
|
||||
result = sessions.reconcile_session_record(path, evidence_root=tmp_path)
|
||||
assert result['status'] == 'IDENTITY_CHANGED'
|
||||
assert result['release_proof'] is None
|
||||
assert child.poll() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kind', ['outside', 'large', 'invalid', 'unknown_protocol'])
|
||||
def test_untrusted_record_is_unconfirmed_not_an_occupancy_fact(tmp_path, kind):
|
||||
owned = tmp_path/'owned'
|
||||
owned.mkdir()
|
||||
path = (tmp_path if kind == 'outside' else owned)/'session.json'
|
||||
path.write_text('x'*65537 if kind == 'large' else '{' if kind == 'invalid' else '{}')
|
||||
result = sessions.reconcile_session_record(path, evidence_root=owned)
|
||||
assert result['status'] == 'UNCONFIRMED'
|
||||
assert result['release_proof'] is None
|
||||
|
||||
|
||||
def test_exited_process_alone_is_not_release_proof(tmp_path, child, monkeypatch):
|
||||
path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))])
|
||||
child.terminate()
|
||||
child.wait(5)
|
||||
def busy(*a, **kw):
|
||||
raise OSError('database still occupied')
|
||||
monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', busy)
|
||||
result = sessions.reconcile_session_record(path, evidence_root=tmp_path)
|
||||
assert result['status'] == 'UNCONFIRMED'
|
||||
assert result['release_proof'] is None
|
||||
assert 'occupied' in result['detail']
|
||||
|
||||
|
||||
def test_release_uses_independent_proof_and_does_not_replay(tmp_path, child, monkeypatch):
|
||||
path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))])
|
||||
child.terminate()
|
||||
child.wait(5)
|
||||
proof = {'status': 'RELEASE_CONFIRMED', 'library': str(tmp_path/'lib.lmc')}
|
||||
monkeypatch.setattr('cbb_refresh_verify.library_lock.release_retained_library', lambda *a, **kw: proof)
|
||||
result = sessions.reconcile_session_record(path, evidence_root=tmp_path)
|
||||
assert result['status'] == 'RELEASED'
|
||||
assert result['release_proof'] == proof
|
||||
|
||||
|
||||
def test_observation_can_stop_without_claiming_native_release(tmp_path, child):
|
||||
import threading
|
||||
path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))])
|
||||
stop = threading.Event()
|
||||
stop.set()
|
||||
result = sessions.observe_retained_session(path, evidence_root=tmp_path, stop=stop)
|
||||
assert result['status'] == 'ACTIVE'
|
||||
assert result['release_proof'] is None
|
||||
assert child.poll() is None
|
||||
|
||||
|
||||
def test_worker_state_cannot_replace_another_operation(tmp_path, child):
|
||||
path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))])
|
||||
sessions.write_session_record(tmp_path/'worker-state.json', {
|
||||
'operation_id': 'different', 'processes': [], 'phase': 'FINISHED'})
|
||||
result = sessions.reconcile_session_record(path, evidence_root=tmp_path)
|
||||
assert result['status'] == 'UNCONFIRMED'
|
||||
assert child.poll() is None
|
||||
|
||||
|
||||
def test_retained_session_yields_global_queue_but_keeps_same_library_waiting(tmp_path, child):
|
||||
import threading
|
||||
from cbb_refresh_verify.operation_queue import NativeQueue
|
||||
from cbb_refresh_verify.session_record import handoff_session, library_available
|
||||
library = tmp_path/'lib.lmc'
|
||||
path = record(tmp_path, [dict(role='worker', **sessions.process_identity(child.pid))])
|
||||
queue = NativeQueue()
|
||||
observation_stop = threading.Event()
|
||||
queue.run(lambda: handoff_session(path, evidence_root=tmp_path, stop=observation_stop))
|
||||
assert library_available(library) is False
|
||||
assert library_available(tmp_path/'other.lmc') is True
|
||||
assert queue.run(lambda: 'other-completed', ready=lambda: library_available(tmp_path/'other.lmc')) == 'other-completed'
|
||||
stop = threading.Event()
|
||||
stop.set()
|
||||
from cbb_refresh_verify.operation_queue import QueueCancelled
|
||||
with pytest.raises(QueueCancelled):
|
||||
queue.run(lambda: pytest.fail('overlapping writer'), stop=stop,
|
||||
ready=lambda: library_available(library))
|
||||
observation_stop.set()
|
||||
|
||||
|
||||
def test_bad_journal_does_not_drop_in_memory_supervision(tmp_path, child):
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
path = tmp_path/'session.json'
|
||||
path.write_text('{')
|
||||
stop = threading.Event()
|
||||
process = SimpleNamespace(poll=child.poll, close=lambda: None, cbb_library=tmp_path/'lib.lmc')
|
||||
sessions.handoff_session(path, evidence_root=tmp_path, process=process, stop=stop)
|
||||
key = str(path.absolute()).casefold()
|
||||
try:
|
||||
time.sleep(.1)
|
||||
with sessions._observer_guard:
|
||||
assert key in sessions._observers, 'active owned handle lost its observer because JSON was unreadable'
|
||||
assert child.poll() is None
|
||||
finally:
|
||||
stop.set()
|
||||
Reference in new issue
Block a user