579 lines
29 KiB
Python
579 lines
29 KiB
Python
"""Resource release proof, never elapsed time, admits later project work."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.trial_task_center import batch_resources, batch_runtime
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
|
|
|
|
def record_at(root, **values):
|
|
session = 'a' * 32
|
|
path = root / 'batch-resources' / session / 'session.json'
|
|
record = dict(version=1, session_id=session, kind='execution', generation=1,
|
|
desktop='dx-' + 'b' * 32, state='blocked', processes=[],
|
|
cad_pid=500, cad_guid='owned', checkpoints={}, retries={}, timings=[])
|
|
record.update(values)
|
|
batch_resources.write_record(path, record)
|
|
return path
|
|
|
|
|
|
def test_restart_reconciliation_checks_cad_identity_even_if_worker_list_empty(tmp_path, monkeypatch):
|
|
path = record_at(tmp_path, cad_process={'pid': 500, 'creation_time': 'original'})
|
|
seen = []
|
|
monkeypatch.setattr(batch_resources, '_process_gone', lambda item: seen.append(item) or False)
|
|
from dx_productivity.trial_task_center import isolated_desktop
|
|
monkeypatch.setattr(isolated_desktop, '_Win32Api', lambda: SimpleNamespace(confirm_desktop_released=lambda _: None))
|
|
assert batch_resources.cleanup_pending_resources(tmp_path)
|
|
assert seen == [{'pid': 500, 'creation_time': 'original'}]
|
|
assert json.loads(path.read_text())['state'] != 'released'
|
|
|
|
|
|
def test_legacy_cad_record_without_process_identity_cannot_be_released(tmp_path, monkeypatch):
|
|
record_at(tmp_path)
|
|
from dx_productivity.trial_task_center import isolated_desktop
|
|
monkeypatch.setattr(isolated_desktop, '_Win32Api', lambda: SimpleNamespace(confirm_desktop_released=lambda _: None))
|
|
monkeypatch.setattr(batch_resources, '_process_gone', lambda item: True)
|
|
assert batch_resources.cleanup_pending_resources(tmp_path)
|
|
|
|
|
|
def test_released_record_without_release_evidence_stays_pending(tmp_path):
|
|
record_at(tmp_path, state='released')
|
|
assert batch_resources.pending_resources(tmp_path)
|
|
|
|
|
|
def test_metadata_only_change_reuses_inventory_but_content_edit_does_not(tmp_path):
|
|
from dx_productivity.trial_task_center.discovery_cache import DiscoveryCache
|
|
from dx_productivity.trial_task_center.quick_run_protocol import ProjectDiscovery
|
|
project = tmp_path / 'project' / 'test.prj'
|
|
project.parent.mkdir()
|
|
project.write_text('project')
|
|
calls = []
|
|
def load():
|
|
calls.append(1)
|
|
return ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})})
|
|
cache = DiscoveryCache(tmp_path / 'data')
|
|
cache.discover(project, ('old',), False, load)
|
|
info = project.stat()
|
|
os.utime(project, ns=(info.st_atime_ns, info.st_mtime_ns + 1000000000))
|
|
cache.discover(project, ('old',), False, load)
|
|
assert len(calls) == 1
|
|
project.write_text('changed')
|
|
cache.discover(project, ('old',), False, load)
|
|
assert len(calls) == 2
|
|
|
|
|
|
def test_runtime_uncertainty_wait_has_owned_exit_observer(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import stop_control, batch_attempt
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', stop_control.StopControl())
|
|
process = SimpleNamespace(poll=lambda: 0)
|
|
runtime.desktop = SimpleNamespace(processes=[process])
|
|
runtime.cad = SimpleNamespace(cad=process)
|
|
monkeypatch.setattr(batch_attempt, 'can_retry_attempt', lambda *args: False)
|
|
captured = {}
|
|
monkeypatch.setattr(stop_control, 'wait_for_resolution', lambda *args, **kw: captured.update(kw))
|
|
runtime._retain_uncertain(tmp_path, None, [('attempt', {})], RuntimeError('lost write'))
|
|
assert captured.get('resolved') is not None
|
|
assert captured['resolved']() is True
|
|
process.poll = lambda: None
|
|
assert captured['resolved']() is False
|
|
|
|
|
|
def test_supervisor_heartbeats_do_not_reset_business_stall_clock():
|
|
from dx_productivity.trial_task_center.batch_supervisor import BusinessProgressWatch
|
|
watch = BusinessProgressWatch(0, idle_timeout=10)
|
|
watch.observe('state', {'phase': 'ready', 'queue': None, 'batch': None}, 1)
|
|
for now in range(2, 12):
|
|
watch.observe('heartbeat', None, now)
|
|
watch.observe('state', {'phase': 'ready', 'queue': None, 'batch': None}, now)
|
|
assert watch.stalled(11)
|
|
watch.observe('progress', 'R2 verified', 12)
|
|
assert not watch.stalled(12)
|
|
|
|
|
|
def test_runtime_read_only_policy_comes_from_operation_provenance(tmp_path):
|
|
assert batch_runtime.BatchCadRuntime(tmp_path, 'bom_import').read_only
|
|
assert batch_runtime.BatchCadRuntime(tmp_path, 'collect_target_inventory').read_only
|
|
assert not batch_runtime.BatchCadRuntime(tmp_path, 'execute_quick_run').read_only
|
|
|
|
|
|
def test_safe_failure_reset_refuses_absent_checkpoint(tmp_path):
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution')
|
|
runtime.desktop = SimpleNamespace(close=lambda: pytest.fail('no release without proof'))
|
|
runtime.cad = object()
|
|
with pytest.raises(TaskCenterError, match='checkpoint|检查点|证明'):
|
|
runtime.reset_project_after_safe_failure(tmp_path / 'board.prj', tmp_path / 'evidence')
|
|
|
|
|
|
def test_unknown_dialog_manual_action_revalidates_full_fingerprint(monkeypatch):
|
|
from dx_productivity.trial_task_center import cad_dialogs
|
|
row = {'pid': 20, 'hwnd': 30, 'class': '#32770', 'title': 'Save?', 'text': 'Save changes?',
|
|
'complete': True, 'buttons': [{'hwnd': 40, 'id': 6, 'text': 'Yes', 'enabled': True}]}
|
|
actions = []
|
|
import win32gui
|
|
monkeypatch.setattr(win32gui, 'PostMessage', lambda *args: actions.append(args))
|
|
monkeypatch.setattr(cad_dialogs, 'inspect_owned_dialogs', lambda *args: (dict(row, text='Other prompt'),))
|
|
with pytest.raises(OSError):
|
|
cad_dialogs.manually_act_on_owned_dialog(None, 'dx-owned', 20, row, 40)
|
|
assert actions == []
|
|
monkeypatch.setattr(cad_dialogs, 'inspect_owned_dialogs', lambda *args: (row,))
|
|
cad_dialogs.manually_act_on_owned_dialog(None, 'dx-owned', 20, row, 40)
|
|
assert actions == [(40, 0x00F5, 0, 0)]
|
|
|
|
|
|
def test_read_only_cad_cleanup_releases_owned_handles_without_write_uncertainty(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
process = SimpleNamespace(pid=50, poll=lambda: None)
|
|
def close():
|
|
process.poll = lambda: 1
|
|
desktop = SimpleNamespace(name='dx-owned', close=close, cleanup_record={
|
|
'desktop_handle_closed': True, 'desktop_confirmation': {'released': True}, 'errors': []})
|
|
session = IsolatedCadSession(tmp_path, desktop=desktop, read_only=True, stop_control=StopControl())
|
|
session.cad = process
|
|
session.binding = SimpleNamespace(environment=lambda: {})
|
|
desktop.start_process = lambda *a, **k: (_ for _ in ()).throw(RuntimeError('read still has a project open'))
|
|
session.__exit__(None, None, None)
|
|
assert process.poll() is not None
|
|
assert session.record['read_only_released'] is True
|
|
assert not (tmp_path / 'uncertain.flag').exists()
|
|
|
|
|
|
def test_read_only_worker_failure_releases_entire_generation_once(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from test_batch_runtime import Cad, Desktop, command_for
|
|
from dx_productivity.trial_task_center import worker_monitor
|
|
from dx_productivity.trial_task_center.project_discovery import ReleasedProjectReadError
|
|
command, _ = command_for(tmp_path)
|
|
calls = []
|
|
def monitor(*args, **kwargs):
|
|
calls.append(kwargs)
|
|
raise subprocess.TimeoutExpired('read', 1)
|
|
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop,
|
|
cad_factory=lambda *args, read_only=False, **kwargs: Cad(*args, **kwargs))
|
|
with runtime:
|
|
with pytest.raises(ReleasedProjectReadError):
|
|
runtime.run_worker(command, 1)
|
|
assert runtime.desktop is runtime.cad is None
|
|
assert runtime.record['state'] == 'released'
|
|
assert len(calls) == 1 and calls[0]['cooperative_abort']
|
|
assert json.loads((Path(command[3]) / 'project-read-release.json').read_text())['resources_released']
|
|
|
|
|
|
def test_new_project_reuses_healthy_cad_after_previous_project_checkpoint(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from test_batch_runtime import Cad, Desktop, command_for, worker_runtime
|
|
first, _ = command_for(tmp_path)
|
|
second, _ = command_for(tmp_path, 'two')
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(first, 1)
|
|
previous = runtime.cad
|
|
runtime.run_worker(second, 1)
|
|
assert not previous.closed
|
|
assert runtime.cad is previous
|
|
assert runtime.generation == 1
|
|
|
|
|
|
def test_owner_loss_requires_bound_parent_identity_and_records_read_policy(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center.read_recovery import check_owner_loss
|
|
owner = {'pid': 40, 'creation_time': 'parent-created'}
|
|
binding = {'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, 'parent_process': owner}
|
|
seen = []
|
|
monkeypatch.setattr(batch_resources, '_process_gone', lambda item: seen.append(item) or True)
|
|
assert check_owner_loss(tmp_path, binding, read_only=True)
|
|
proof = json.loads((tmp_path / 'owner-loss.json').read_text())
|
|
assert proof['parent_process'] == owner and proof['read_only'] is True
|
|
assert seen == [owner]
|
|
with pytest.raises(ValueError):
|
|
check_owner_loss(tmp_path, dict(binding, parent_process={'pid': 40}), read_only=True)
|
|
|
|
|
|
def test_unsafe_worker_exception_wait_resolves_when_owned_worker_exits(tmp_path, monkeypatch):
|
|
import io
|
|
import subprocess
|
|
from dx_productivity.trial_task_center import worker_monitor, stop_control
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
process = SimpleNamespace(pid=123, stdout=io.BytesIO(), stderr=io.BytesIO(), returncode=None,
|
|
poll=lambda: None, close=lambda: None)
|
|
class Owned:
|
|
def __init__(self, *args): pass
|
|
def close(self): pass
|
|
def stop_workers(self, _): return []
|
|
monkeypatch.setattr(worker_monitor, '_OwnedWorkers', Owned)
|
|
real_atomic = worker_monitor._atomic_text
|
|
def fail_ready(path, text):
|
|
if path.suffix == '.ready':
|
|
# An actual launched worker can already be inside an unsafe stage.
|
|
raise OSError('ready evidence locked')
|
|
return real_atomic(path, text)
|
|
monkeypatch.setattr(worker_monitor, '_atomic_text', fail_ready)
|
|
original = worker_monitor.ProgressWatch
|
|
class UnsafeWatch(original):
|
|
def __init__(self, *args):
|
|
super().__init__(*args)
|
|
self.safe_stop = False
|
|
monkeypatch.setattr(worker_monitor, 'ProgressWatch', UnsafeWatch)
|
|
captured = {}
|
|
def resolve(control, root, error, **options):
|
|
captured.update(options)
|
|
process.poll = lambda: 1
|
|
process.returncode = 1
|
|
assert options['resolved']()
|
|
monkeypatch.setattr(stop_control, 'wait_for_resolution', resolve)
|
|
process.wait = lambda **options: 1
|
|
with pytest.raises(worker_monitor.WorkerMonitorError):
|
|
worker_monitor.run_monitored_process(('cscript', '', '', str(tmp_path), 'python', 'job'), 1,
|
|
process_factory=lambda *a, **kw: process, force_control=StopControl())
|
|
assert captured.get('resolved') is not None
|
|
|
|
|
|
def test_manual_dialog_transport_is_generation_bound_and_one_shot(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_worker import consume_manual_dialog_actions
|
|
binding = {'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, 'generation': 2, 'session_token': 'token'}
|
|
calls = []
|
|
runtime = SimpleNamespace(generation=2, cad=SimpleNamespace(token='token'),
|
|
manual_dialog_action=lambda dialog, button: calls.append((dialog, button)))
|
|
request = dict(binding, version=1, request_id='c' * 32, dialog={'pid': 1}, button_hwnd=2)
|
|
batch_resources.write_record(tmp_path / ('manual-dialog-' + request['request_id'] + '.json'), request)
|
|
consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id'])
|
|
consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id'])
|
|
assert calls == [({'pid': 1}, 2)]
|
|
stale = dict(request, generation=1, request_id='d' * 32)
|
|
batch_resources.write_record(tmp_path / ('manual-dialog-' + stale['request_id'] + '.json'), stale)
|
|
consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id'])
|
|
assert calls == [({'pid': 1}, 2)]
|
|
|
|
|
|
def test_cache_hashes_after_verified_reader_release(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center import discovery_cache
|
|
from dx_productivity.trial_task_center.quick_run_protocol import ProjectDiscovery
|
|
project = tmp_path / 'board.prj'
|
|
project.write_text('project')
|
|
calls, locked = [], [False]
|
|
real_fingerprint = discovery_cache.project_fingerprint
|
|
def fingerprint(*args, **kwargs):
|
|
assert not locked[0], 'fingerprint attempted before read generation release'
|
|
return real_fingerprint(*args, **kwargs)
|
|
def load():
|
|
calls.append('load')
|
|
locked[0] = True
|
|
return ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})})
|
|
def release():
|
|
calls.append('release')
|
|
locked[0] = False
|
|
monkeypatch.setattr(discovery_cache, 'project_fingerprint', fingerprint)
|
|
cache = discovery_cache.DiscoveryCache(tmp_path / 'data')
|
|
cache.discover(project, ('old',), False, load, release_idle=release)
|
|
cache.discover(project, ('old',), False, load, release_idle=release)
|
|
assert calls == ['load', 'release']
|
|
|
|
|
|
def test_cleanup_error_never_drops_live_write_cad_or_closes_its_desktop(tmp_path):
|
|
from test_batch_runtime import Desktop
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
|
runtime._ensure_desktop()
|
|
process = SimpleNamespace(poll=lambda: None)
|
|
def refuse(*args):
|
|
raise WorkerMonitorError('CAD still saving; evidence write failed')
|
|
cad = SimpleNamespace(cad=process, __exit__=refuse, record={})
|
|
runtime.cad = cad
|
|
with pytest.raises(TaskCenterError):
|
|
runtime.close()
|
|
assert runtime.cad is cad
|
|
assert not runtime.desktop.released
|
|
# Leave no retained live registry state in the test process.
|
|
batch_resources.LIVE_RESOURCES.pop(str(runtime.path), None)
|
|
|
|
|
|
def test_owned_exit_observer_waits_for_job_descendants(tmp_path):
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution')
|
|
process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: False)
|
|
runtime.desktop = SimpleNamespace(processes=[process])
|
|
assert not runtime.owned_processes_exited()
|
|
process.tree_exited = lambda: True
|
|
assert runtime.owned_processes_exited()
|
|
|
|
|
|
def test_startup_dialog_owner_is_published_before_guid_binding(tmp_path):
|
|
from test_batch_runtime import Desktop
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
|
with runtime:
|
|
with runtime.native_scope(tmp_path / 'startup') as lease:
|
|
process = lease.start_process(('viewdraw.exe',))
|
|
lease.register_cad_process(process, 'session-token')
|
|
record = json.loads(runtime.path.read_text())
|
|
assert record['cad_pid'] == process.pid
|
|
assert record['cad_process']['pid'] == process.pid
|
|
assert record['cad_token'] == 'session-token'
|
|
assert record['cad_guid'] is None
|
|
|
|
|
|
def test_native_read_failure_releases_without_repeating_project(tmp_path):
|
|
import hashlib
|
|
from test_batch_runtime import Desktop
|
|
from dx_productivity.trial_task_center.project_discovery import ReleasedProjectReadError
|
|
project, database = tmp_path / 'board.prj', tmp_path / 'database.dat'
|
|
project.write_text('project')
|
|
database.write_text('database')
|
|
evidence = tmp_path / 'native'
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop)
|
|
with runtime:
|
|
with runtime.native_scope(evidence) as scope:
|
|
pass
|
|
batch_resources.write_record(evidence / 'native-retry-proof.json', {'version': 1,
|
|
'native_read_only': True, 'workers_closed': True, 'binding': scope.identity,
|
|
'project_path': str(project), 'evidence_root': str(evidence),
|
|
'source_hashes': {str(item): hashlib.sha256(item.read_bytes()).hexdigest() for item in (project, database)}})
|
|
with pytest.raises(ReleasedProjectReadError):
|
|
runtime.recover_native_failure(project, evidence, 'read timed out')
|
|
assert runtime.generation == 1 and runtime.record['state'] == 'released'
|
|
assert runtime.desktop is runtime.cad is None
|
|
assert runtime.record['retries'] == {}
|
|
|
|
|
|
@pytest.mark.parametrize('native', [False, True])
|
|
def test_repeated_live_frames_cannot_hide_business_stall(native):
|
|
from dx_productivity.trial_task_center.worker_monitor import ProgressWatch
|
|
watch = ProgressWatch('token', 'job', 10, 0)
|
|
stage = 'native_snapshot_read' if native else 'native_replace'
|
|
assert watch.accept(f'DX_PROGRESS\ttoken\tjob\t1\t{stage}\t0\t10\t0\n'.encode(), 1)
|
|
for sequence in range(2, 12):
|
|
if native:
|
|
assert watch.accept_native(f'DX_NATIVE_PROGRESS\ttoken\tjob\t1\tjob\t{sequence}\t0\t10\t0\n'.encode(), sequence)
|
|
else:
|
|
assert watch.accept(f'DX_PROGRESS\ttoken\tjob\t{sequence}\t{stage}\t0\t10\t0\n'.encode(), sequence)
|
|
assert watch.action(11) == 'request_stop'
|
|
assert watch.action(21) == 'await_safe_completion'
|
|
assert watch.action(100) == 'failure_cleanup' # Failure cleanup still requires durable uncertainty protection.
|
|
|
|
|
|
def test_finished_supervisor_label_does_not_hide_a_live_original_process(tmp_path, monkeypatch):
|
|
from dx_productivity.trial_task_center.batch_supervisor import pending_supervisors
|
|
batch_id = 'a' * 32
|
|
batch_resources.write_record(tmp_path / 'batch-resources' / batch_id / 'supervisor.json', {
|
|
'version': 1, 'batch_id': batch_id, 'attempt_id': 'b' * 32,
|
|
'operation': 'execute_task_queue', 'state': 'finished', 'task_ids': ['A'],
|
|
'process': {'pid': 20, 'creation_time': 'worker'},
|
|
'parent_process': {'pid': 10, 'creation_time': 'parent'}, 'request_sha256': 'f' * 64})
|
|
monkeypatch.setattr(batch_resources, '_process_gone', lambda identity: False)
|
|
assert pending_supervisors(tmp_path)
|
|
|
|
|
|
def test_crashed_write_supervisor_releases_lingering_descendant_after_protection(tmp_path, monkeypatch):
|
|
import sys
|
|
from dx_productivity.trial_task_center import batch_supervisor
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
monkeypatch.setattr(batch_supervisor, '_command', lambda request: (sys.executable, '-c',
|
|
'import subprocess,sys; subprocess.Popen([sys.executable,"-c","import time;time.sleep(6)"], '
|
|
'stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)'))
|
|
store = TaskStore(tmp_path / 'tasks.db')
|
|
try:
|
|
service = TaskCenterService(store, configure_logging(tmp_path))
|
|
task = store.create_task('harmless owned process probe')
|
|
with pytest.raises(TaskCenterError) as caught:
|
|
batch_supervisor.run_supervised(service, 'execute_quick_run', {'task_id': task.task_id},
|
|
stop_event=StopControl())
|
|
record = json.loads(next((tmp_path / 'batch-resources').glob('*/process.json')).read_text())
|
|
assert record['cleanup']['forced_termination'] is True
|
|
assert caught.value.resources_released
|
|
assert list(tmp_path.rglob('uncertain.flag'))
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def test_failed_write_session_creates_failure_control_and_releases_owned_process(tmp_path, monkeypatch):
|
|
import sys
|
|
from uuid import uuid4
|
|
from dx_productivity.trial_task_center import isolated_cad
|
|
from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop
|
|
desktop = PrivateDesktop('dx-' + uuid4().hex, enabled=True)
|
|
desktop.__enter__()
|
|
process = desktop.start_process((sys.executable, '-c', 'import time;time.sleep(60)'))
|
|
session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop)
|
|
session.cad = process
|
|
session.binding = SimpleNamespace(environment=lambda: {})
|
|
session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process))
|
|
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: (sys.executable, '-c', 'raise SystemExit(1)'))
|
|
try:
|
|
with pytest.raises(isolated_cad.WorkerMonitorError):
|
|
session.__exit__(None, None, None)
|
|
assert process.poll() is not None
|
|
assert session.record['cleanup_mode'] == 'mentor_release_required'
|
|
assert (tmp_path / 'uncertain.flag').exists()
|
|
assert batch_resources.pending_resources(tmp_path) == ()
|
|
assert str(session.root) not in isolated_cad.RETAINED_CAD_SESSIONS
|
|
finally:
|
|
process.terminate(reason='explicit harmless test cleanup')
|
|
desktop.close()
|
|
getattr(isolated_cad, 'RETAINED_CAD_SESSIONS', {}).pop(str(session.root), None)
|
|
|
|
|
|
def test_force_pending_owned_job_requires_confirmation_identity_and_durable_protection(tmp_path, monkeypatch):
|
|
import os
|
|
import sys
|
|
from uuid import uuid4
|
|
import win32api
|
|
from dx_productivity.trial_task_center import batch_supervisor
|
|
from dx_productivity.trial_task_center.isolated_desktop import _Win32Api
|
|
from dx_productivity.trial_task_center.batch_resources import force_pending_owned_resources
|
|
api = _Win32Api()
|
|
process = api.create_process((sys.executable, '-c', 'import time;time.sleep(60)'), None, None, None)
|
|
batch_id = uuid4().hex
|
|
path = tmp_path / 'batch-resources' / batch_id / 'supervisor.json'
|
|
parent = SimpleNamespace(pid=os.getpid(), handle=win32api.GetCurrentProcess())
|
|
journal = {'version': 1, 'batch_id': batch_id, 'attempt_id': uuid4().hex,
|
|
'operation': 'execute_quick_run', 'state': 'blocked', 'task_ids': ['A'],
|
|
'process': batch_resources.process_identity(process), 'parent_process': batch_resources.process_identity(parent),
|
|
'request_sha256': 'f' * 64}
|
|
batch_resources.write_record(path, journal)
|
|
batch_supervisor._LIVE_SUPERVISORS[path] = (api, process)
|
|
try:
|
|
with pytest.raises(TaskCenterError):
|
|
force_pending_owned_resources(tmp_path)
|
|
assert process.poll() is None and not list(tmp_path.rglob('uncertain.flag'))
|
|
corrupt = dict(journal, process=dict(journal['process'], creation_time='reused PID'))
|
|
batch_resources.write_record(path, corrupt)
|
|
with pytest.raises(TaskCenterError):
|
|
force_pending_owned_resources(tmp_path, confirmed=True)
|
|
assert process.poll() is None
|
|
batch_resources.write_record(path, journal)
|
|
mark = batch_supervisor._mark_interrupted
|
|
monkeypatch.setattr(batch_supervisor, '_mark_interrupted', lambda *args: (_ for _ in ()).throw(PermissionError('protection denied')))
|
|
with pytest.raises(PermissionError):
|
|
force_pending_owned_resources(tmp_path, confirmed=True)
|
|
assert process.poll() is None
|
|
monkeypatch.setattr(batch_supervisor, '_mark_interrupted', mark)
|
|
assert force_pending_owned_resources(tmp_path, confirmed=True) == ()
|
|
assert process.poll() is not None
|
|
assert list(tmp_path.rglob('uncertain.flag'))
|
|
finally:
|
|
process.terminate(reason='explicit harmless test cleanup')
|
|
process.close()
|
|
api.close_pending()
|
|
batch_supervisor._LIVE_SUPERVISORS.pop(path, None)
|
|
|
|
|
|
def test_safe_failure_reset_returns_bound_prewrite_source_and_release_proof(tmp_path, monkeypatch):
|
|
import hashlib
|
|
import subprocess
|
|
from test_batch_runtime import command_for, worker_runtime
|
|
command, project = command_for(tmp_path)
|
|
runtime = worker_runtime(tmp_path, monkeypatch, lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', ''))
|
|
with runtime:
|
|
runtime.run_worker(command, 1)
|
|
proof = runtime.reset_project_after_safe_failure(project, Path(command[3]))
|
|
assert proof['no_mutation'] is True and proof['source_unchanged'] is True
|
|
assert proof['resources_released'] is True and proof['project_closed'] is True
|
|
assert proof['job_id'] == 'job' and proof['attempt_id'] == runtime.record['active_attempt']
|
|
assert proof['session_id'] == runtime.session_id and proof['generation'] == 1
|
|
assert proof['project_path'] == str(project.resolve())
|
|
assert proof['attempt_record_sha256'] == hashlib.sha256(Path(proof['attempt_record_path']).read_bytes()).hexdigest()
|
|
assert json.loads(Path(proof['proof_path']).read_text()) == proof
|
|
|
|
|
|
def test_discovery_identity_preserves_actual_metadata_unknown_vs_empty():
|
|
from dx_productivity.trial_task_center.discovery_cache import _discovery_identity
|
|
row = SimpleNamespace(key='uid', refdes='R1', old_part_number='old', old_value='value',
|
|
actual_part_label=None, actual_cell=None)
|
|
discovery = SimpleNamespace(sheets=(('MAIN', '1'),), matched_old_parts={('MAIN', '1'): frozenset({'old'})}, components=(row,))
|
|
before = _discovery_identity(discovery)
|
|
row.actual_part_label = ''
|
|
assert _discovery_identity(discovery) != before
|
|
|
|
|
|
def test_force_pending_runtime_uses_live_handles_and_preserves_write_uncertainty(tmp_path):
|
|
import sys
|
|
from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession
|
|
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution')
|
|
scope = runtime.native_scope(tmp_path / 'quick-run')
|
|
scope.__enter__()
|
|
process = scope.start_process((sys.executable, '-c', 'import time;time.sleep(60)'))
|
|
session = IsolatedCadSession(tmp_path / 'cad', desktop=scope)
|
|
session.cad = process
|
|
session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process))
|
|
runtime.cad = session
|
|
runtime._active_evidence = tmp_path / 'quick-run'
|
|
runtime._save(cad_pid=process.pid, cad_guid='owned', cad_process=batch_resources.process_identity(process))
|
|
try:
|
|
assert batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) == ()
|
|
assert process.poll() is not None
|
|
assert runtime.record['state'] == 'released'
|
|
assert (tmp_path / 'quick-run' / 'uncertain.flag').exists()
|
|
assert runtime.record['force_confirmed'] is True
|
|
finally:
|
|
process.terminate(reason='explicit harmless test cleanup')
|
|
runtime.cad = None
|
|
runtime.close(reason='manual_retry')
|
|
|
|
|
|
def test_failed_session_releases_once_and_keeps_uncertainty_after_recheck(tmp_path, monkeypatch):
|
|
import sys
|
|
import time
|
|
from uuid import uuid4
|
|
from dx_productivity.trial_task_center import isolated_cad
|
|
from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop
|
|
desktop = PrivateDesktop('dx-' + uuid4().hex, enabled=True)
|
|
desktop.__enter__()
|
|
process = desktop.start_process((sys.executable, '-c', 'import time;time.sleep(1.5)'))
|
|
session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop)
|
|
session.cad = process
|
|
session.binding = SimpleNamespace(environment=lambda: {})
|
|
session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process))
|
|
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: (sys.executable, '-c', 'raise SystemExit(1)'))
|
|
try:
|
|
with pytest.raises(isolated_cad.WorkerMonitorError):
|
|
session.__exit__(None, None, None)
|
|
assert process.wait(timeout=4) != 0
|
|
deadline = time.monotonic() + 2
|
|
while not all(item.tree_exited() for item in session._owned_processes()):
|
|
assert time.monotonic() < deadline
|
|
time.sleep(.01)
|
|
assert batch_resources.cleanup_pending_resources(tmp_path) == ()
|
|
assert (tmp_path / 'uncertain.flag').exists()
|
|
session.__exit__(None, None, None)
|
|
assert process.cleanup_record['forced_termination'] is True
|
|
finally:
|
|
process.terminate(reason='explicit harmless test cleanup')
|
|
desktop.close()
|
|
isolated_cad.RETAINED_CAD_SESSIONS.pop(str(session.root), None)
|
|
|
|
|
|
def test_failed_session_releases_descendants_even_after_root_exit(tmp_path):
|
|
from dx_productivity.trial_task_center import isolated_cad
|
|
process = SimpleNamespace(pid=30, poll=lambda: 0, tree_exited=lambda: False)
|
|
closed = []
|
|
def close():
|
|
closed.append(True)
|
|
process.tree_exited = lambda: True
|
|
desktop = SimpleNamespace(name='dx-owned', _processes=[process], close=close)
|
|
session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop)
|
|
session.cad = process
|
|
try:
|
|
with pytest.raises(isolated_cad.WorkerMonitorError):
|
|
session.__exit__(None, None, None)
|
|
assert closed == [True]
|
|
assert session.record['release_state'] == 'released'
|
|
assert session.record['cleanup_mode'] == 'mentor_release_required'
|
|
finally:
|
|
isolated_cad.RETAINED_CAD_SESSIONS.pop(str(session.root), None)
|
|
|
|
|
|
def test_repeat_explicit_force_checks_whole_job_not_only_completed_root(tmp_path):
|
|
from dx_productivity.trial_task_center.batch_supervisor import _force_owned_supervisor
|
|
calls = []
|
|
process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: False,
|
|
terminate=lambda: calls.append('force-job'))
|
|
journal = {'version': 1, 'batch_id': 'a' * 32, 'attempt_id': 'b' * 32,
|
|
'operation': 'execute_quick_run', 'task_ids': ['A'], 'force_state': 'termination_requested'}
|
|
_force_owned_supervisor(tmp_path, journal, process)
|
|
assert calls == ['force-job']
|