293 lines
12 KiB
Python
293 lines
12 KiB
Python
"""AI observation must leave the desktop execution lane available."""
|
|
from threading import Event
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from PySide6.QtCore import QThreadPool
|
|
|
|
from dx_productivity.trial_task_center.ui.workers import ServiceWorker
|
|
from .test_dsh_diagnosis_lifecycle import slow_diagnosis
|
|
from .test_dsh_operations import setup_view
|
|
|
|
|
|
pytestmark = pytest.mark.usefixtures('ai_ready')
|
|
|
|
|
|
@pytest.fixture
|
|
def single_execution_thread(ui, qtbot):
|
|
ui.settle()
|
|
pool = QThreadPool.globalInstance()
|
|
previous = pool.maxThreadCount()
|
|
pool.setMaxThreadCount(1)
|
|
yield pool
|
|
qtbot.waitUntil(lambda: pool.activeThreadCount() == 0, timeout=5000)
|
|
pool.setMaxThreadCount(previous)
|
|
|
|
|
|
@pytest.mark.parametrize('source', ['timer', 'history'])
|
|
def test_operations_observation_never_reserves_or_starves_task_execution(
|
|
ui, monkeypatch, qtbot, single_execution_thread, source):
|
|
window = ui.window
|
|
view = setup_view(ui, monkeypatch)
|
|
operations = window._dsh_operations
|
|
started, release, executed = Event(), Event(), Event()
|
|
|
|
def read(**_kwargs):
|
|
started.set()
|
|
assert release.wait(5), 'Controlled operations read was not released'
|
|
return {'schedules': []}
|
|
|
|
monkeypatch.setattr(operations, '_read', read)
|
|
if source == 'timer':
|
|
operations.tick()
|
|
else:
|
|
operations.request({'request_id': 'a' * 32, 'spec': {'operation': 'history'}})
|
|
try:
|
|
qtbot.waitUntil(started.is_set, timeout=5000)
|
|
assert operations.worker not in window._workers
|
|
assert window.task_list.new_button.isEnabled()
|
|
window._run_async(executed.set, lambda _: None)
|
|
qtbot.waitUntil(executed.is_set, timeout=1000)
|
|
assert not release.is_set(), 'Task work must start before the AI read finishes'
|
|
finally:
|
|
release.set()
|
|
qtbot.waitUntil(lambda: operations.worker is None and not window._workers, timeout=5000)
|
|
if source == 'history':
|
|
assert view.lifecycle_results[-1]['status'] == 'read'
|
|
|
|
|
|
def test_availability_check_cannot_starve_task_worker(
|
|
ui, monkeypatch, qtbot, single_execution_thread):
|
|
from dx_productivity.trial_task_center.ui import dsh_status
|
|
|
|
status = ui.window._dsh_status
|
|
status.timer.stop()
|
|
qtbot.waitUntil(lambda: not status.query.busy, timeout=5000)
|
|
started, release, executed = Event(), Event(), Event()
|
|
|
|
def check(_root):
|
|
started.set()
|
|
assert release.wait(5), 'Controlled availability check was not released'
|
|
return {'available': True, 'code': 'available', 'message': 'Controlled readiness'}
|
|
|
|
monkeypatch.setattr(dsh_status, 'check_availability', check)
|
|
status.refresh()
|
|
worker = ServiceWorker(executed.set)
|
|
try:
|
|
qtbot.waitUntil(started.is_set, timeout=5000)
|
|
single_execution_thread.start(worker)
|
|
qtbot.waitUntil(executed.is_set, timeout=1000)
|
|
assert not release.is_set()
|
|
finally:
|
|
release.set()
|
|
qtbot.waitUntil(lambda: not status.query.busy, timeout=5000)
|
|
|
|
|
|
@pytest.mark.parametrize('kind', ['authorized_retry', 'resource_cleanup'])
|
|
def test_model_wait_never_owns_task_controls(slow_diagnosis, ui, qtbot, kind):
|
|
window, automation, _, start = slow_diagnosis
|
|
if kind == 'resource_cleanup':
|
|
start(read_only=False, cleanup=True)
|
|
else:
|
|
automation.enqueue(window._task_id, 'execution_failed', 'Controlled authorized failure',
|
|
execution_authorized=True, run_id='b' * 32)
|
|
automation._drain()
|
|
qtbot.waitUntil(lambda: bool(window._dsh_view.requests), timeout=5000)
|
|
try:
|
|
assert automation.worker not in window._workers
|
|
assert window._batch_stop is None
|
|
assert window._active_quick_run_task_id is None
|
|
assert window.task_list.new_button.isEnabled()
|
|
finally:
|
|
automation.cancel()
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
ui.settle()
|
|
|
|
|
|
def test_finishing_old_diagnosis_preserves_new_manual_run(slow_diagnosis, ui, qtbot):
|
|
window, automation, _, _ = slow_diagnosis
|
|
automation.enqueue(window._task_id, 'execution_failed', 'Controlled old failure',
|
|
execution_authorized=True, run_id='c' * 32)
|
|
automation._drain()
|
|
qtbot.waitUntil(lambda: bool(window._dsh_view.requests), timeout=5000)
|
|
owner = ui.tasks[1].task_id
|
|
window._begin_run(owner)
|
|
manual_stop = window._batch_stop
|
|
manual_run = window._dsh_run_id
|
|
try:
|
|
automation.cancel()
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
assert window._batch_stop is manual_stop
|
|
assert window._active_quick_run_task_id == owner
|
|
assert window._dsh_run_id == manual_run
|
|
assert window._dsh_live_run['active']
|
|
assert not manual_stop.is_set()
|
|
finally:
|
|
window._finish_run()
|
|
ui.settle()
|
|
|
|
|
|
def reply_to_model(view, action='none'):
|
|
request = view.requests[-1]
|
|
view.diagnosis_event.emit({'status': 'started', 'request_id': request['request_id'],
|
|
'session_id': 'controlled-session'})
|
|
view.diagnosis_event.emit({'status': 'completed', 'request_id': request['request_id'],
|
|
'session_id': 'controlled-session',
|
|
'reply': '{"summary":"Controlled diagnosis","action":"' + action + '"}'})
|
|
|
|
|
|
def test_cleanup_reserves_lane_only_during_service_action(
|
|
slow_diagnosis, monkeypatch, qtbot):
|
|
from dx_productivity.trial_task_center.ui import dsh_automation
|
|
|
|
window, automation, _, start = slow_diagnosis
|
|
started, release = Event(), Event()
|
|
pending = [{'state': 'blocked', 'kind': 'controlled_test_resource'}]
|
|
publications = []
|
|
|
|
def cleanup():
|
|
started.set()
|
|
assert release.wait(5), 'Controlled cleanup was not released'
|
|
return tuple(pending)
|
|
|
|
def publish(*_args):
|
|
publications.append(automation.worker in window._workers)
|
|
return {'status': 'unavailable'}
|
|
|
|
monkeypatch.setattr(window.service, 'pending_batch_resources', lambda: tuple(pending))
|
|
monkeypatch.setattr(window.service, 'cleanup_batch_resources', cleanup)
|
|
monkeypatch.setattr(dsh_automation, 'GitIssuePublisher', lambda _path: publish)
|
|
view = start(read_only=False, cleanup=True)
|
|
reply_to_model(view, 'cleanup_resources')
|
|
try:
|
|
qtbot.waitUntil(started.is_set, timeout=5000)
|
|
assert automation.worker in window._workers
|
|
assert automation._owns_execution_lane
|
|
assert window._batch_stop is None
|
|
finally:
|
|
release.set()
|
|
qtbot.waitUntil(lambda: len(view.requests) == 2, timeout=5000)
|
|
assert view.requests[-1]['phase'] == 'reflect'
|
|
assert automation.worker not in window._workers
|
|
assert not automation._owns_execution_lane
|
|
assert window.task_list.new_button.isEnabled()
|
|
pending.clear()
|
|
reply_to_model(view)
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
assert publications == [False]
|
|
|
|
|
|
@pytest.mark.parametrize('action', ['cleanup_resources', 'retry_failed'])
|
|
def test_new_manual_run_denies_old_ai_mutation_without_touching_controls(
|
|
slow_diagnosis, ui, monkeypatch, qtbot, action):
|
|
from dx_productivity.trial_task_center.ui import dsh_automation
|
|
|
|
window, automation, published, start = slow_diagnosis
|
|
mutations = []
|
|
receipt_started, release_receipt = Event(), Event()
|
|
monkeypatch.setattr(window.service, 'cleanup_batch_resources', lambda: mutations.append('cleanup'))
|
|
monkeypatch.setattr(window.service, 'prepare_failed_quick_run', lambda _task: mutations.append('prepare'))
|
|
if action == 'cleanup_resources':
|
|
view = start(read_only=False, cleanup=True)
|
|
else:
|
|
context = SimpleNamespace(task=SimpleNamespace(status=SimpleNamespace(value='running')),
|
|
interrupted=False, phase='retry')
|
|
monkeypatch.setattr(dsh_automation.IncidentRunner, '_snapshot',
|
|
lambda _self, task_id, _code, _message: ({'task_id': task_id, 'status': 'running'}, context))
|
|
def certified_receipt(_task):
|
|
receipt_started.set()
|
|
assert release_receipt.wait(5), 'Controlled receipt read was not released'
|
|
return {'items': [{'auto_retries_remaining': 1}]}
|
|
monkeypatch.setattr(window.service, 'execution_problem_overview', certified_receipt)
|
|
automation.enqueue(window._task_id, 'execution_failed', 'Controlled retryable failure',
|
|
execution_authorized=True, run_id='d' * 32)
|
|
automation._drain()
|
|
# Certified replay precedes diagnosis; change the GUI execution epoch
|
|
# while collecting its receipt, before action admission or any mutation.
|
|
qtbot.waitUntil(receipt_started.is_set, timeout=5000)
|
|
view = window._dsh_view
|
|
window._begin_run(ui.tasks[1].task_id)
|
|
manual_stop = window._batch_stop
|
|
owner = window._active_quick_run_task_id
|
|
try:
|
|
release_receipt.set()
|
|
qtbot.waitUntil(lambda: window._dsh_view is not None and bool(window._dsh_view.requests), timeout=5000)
|
|
view = window._dsh_view
|
|
reply_to_model(view, action)
|
|
qtbot.waitUntil(lambda: len(view.requests) == 2, timeout=5000)
|
|
assert not mutations
|
|
assert automation.worker not in window._workers
|
|
assert window._batch_stop is manual_stop
|
|
assert window._active_quick_run_task_id == owner
|
|
assert not manual_stop.is_set()
|
|
assert 'ai_action_context_changed' in str(automation.latest)
|
|
assert not list(ui.data.glob('dsh/retry-budget/*.json'))
|
|
reply_to_model(view)
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
assert window._batch_stop is manual_stop
|
|
assert window._active_quick_run_task_id == owner
|
|
assert len(published) == 1
|
|
finally:
|
|
release_receipt.set()
|
|
automation.cancel()
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
window._finish_run()
|
|
ui.settle()
|
|
|
|
|
|
def test_completed_taskless_observation_does_not_start_another_model_run(
|
|
slow_diagnosis, ui, qtbot):
|
|
window, automation, published, start = slow_diagnosis
|
|
view = start()
|
|
reply_to_model(view)
|
|
qtbot.waitUntil(lambda: len(view.requests) == 2, timeout=5000)
|
|
reply_to_model(view)
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
assert ('', 'operation_stalled', 'observation') in automation.seen
|
|
assert automation.enqueue('', 'operation_stalled', 'same persistent global error', read_only=True)
|
|
automation._drain()
|
|
qtbot.wait(20)
|
|
assert len(view.requests) == 2
|
|
assert automation.worker is None
|
|
assert not automation.pending
|
|
assert len(published) == 1
|
|
automation.set_enabled(False)
|
|
automation.set_enabled(True)
|
|
assert automation.enqueue('', 'operation_stalled', 'explicit new episode', read_only=True)
|
|
automation._drain()
|
|
qtbot.waitUntil(lambda: len(view.requests) == 3, timeout=5000)
|
|
automation.cancel()
|
|
qtbot.waitUntil(lambda: automation.worker is None, timeout=5000)
|
|
ui.settle()
|
|
|
|
|
|
def test_admission_timeout_releases_even_if_gui_just_admitted(ui, monkeypatch):
|
|
from dx_productivity.trial_task_center.ui import dsh_automation as module
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
|
|
ui.settle()
|
|
window = ui.window
|
|
automation = module.DshAutomation(window)
|
|
automation.worker = object()
|
|
automation.current_request = module.IncidentRequest('', 'batch_resources_pending', '',
|
|
False, 'observation', cleanup_allowed=True)
|
|
automation._observed_run_id = window._dsh_run_id
|
|
automation._action_stop = StopControl()
|
|
|
|
class ExpiringEvent:
|
|
def set(self):
|
|
pass
|
|
def wait(self, _timeout=None):
|
|
return False
|
|
|
|
monkeypatch.setattr(module, 'Event', ExpiringEvent)
|
|
try:
|
|
with pytest.raises(ValueError, match='admission_timeout'):
|
|
with automation._action_scope('cleanup_resources'):
|
|
pytest.fail('Timed-out admission must not mutate')
|
|
assert not automation._owns_execution_lane
|
|
assert automation.worker not in window._workers
|
|
assert automation._admitted_action is None
|
|
finally:
|
|
automation.worker = None
|