258 lines
14 KiB
Python
258 lines
14 KiB
Python
"""Startup pause is a bound no-dispatch outcome, never an error-name heuristic."""
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
from threading import Event
|
|
|
|
import pytest
|
|
from test_nonblocking_items import component_case
|
|
|
|
|
|
def test_session_pause_requires_current_in_memory_session_not_rewritten_request(tmp_path, monkeypatch):
|
|
import subprocess
|
|
from dx_productivity.trial_task_center.project_session import ProjectSessionWorker
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
session = ProjectSessionWorker(tmp_path / 'board.prj', tmp_path)
|
|
def forged(command, timeout):
|
|
foreign = 'f' * 32
|
|
(tmp_path / 'project-session.request').write_text(f'{foreign}\t{session.project}\tjob')
|
|
session._result = subprocess.CompletedProcess(command, 74, '', '')
|
|
session._result.startup_pause = {'session_token': foreign, 'job_id': 'job',
|
|
'project_path': str(session.project), 'worker_tree_exited': True}
|
|
session._done.set()
|
|
monkeypatch.setattr(session, '_monitor', forged)
|
|
with pytest.raises(WorkerMonitorError, match='different project session'):
|
|
session.run_step(('worker', '', '', str(tmp_path), '', '', 'job'), 2)
|
|
session._thread.join()
|
|
|
|
|
|
@pytest.mark.skipif(os.name != 'nt', reason='Extracted actual VBS producer and native owned worker')
|
|
def test_real_vbs_startup_pause_produces_correlated_no_project_ack(tmp_path):
|
|
from dx_productivity.trial_task_center.worker_monitor import run_monitored_process
|
|
from dx_productivity.trial_task_center.isolated_desktop import _Win32Api
|
|
source = (Path(__file__).resolve().parents[2] / 'integration/dxdesigner795/QuickPartReplace.vbs').read_text(encoding='utf-8')
|
|
progress = re.search(r'(?ms)^Sub QPProgress\(.*?^End Sub', source).group()
|
|
reader = re.search(r'(?ms)^Function QPReadUtf8\(.*?^End Function', source).group()
|
|
session = 'a' * 32
|
|
project = tmp_path / 'board.prj'
|
|
project.write_text('never opened')
|
|
(tmp_path / 'project-session.request').write_text(f'{session}\t{project}\tjob', encoding='utf-8')
|
|
script = tmp_path / 'startup.vbs'
|
|
script.write_text('''Dim QP_AttemptToken, QP_Fso, QP_Root, QP_ExpectedJobId, QP_MonitorJobId
|
|
Dim QP_SessionMode, QP_SessionToken, QP_SessionProject, QP_App, QP_OwnedProject, QP_OwnedProjectPath
|
|
Dim QP_NativeMutationPossible, QP_RestoreTransactionOpen, QP_NativeCallActive, QP_ProgressSequence
|
|
Set QP_Fso=CreateObject("Scripting.FileSystemObject")
|
|
Set QP_App=Nothing
|
|
QP_Root=WScript.Arguments(0)
|
|
QP_ExpectedJobId=WScript.Arguments(3)
|
|
QP_MonitorJobId=QP_ExpectedJobId
|
|
QP_AttemptToken=WScript.Arguments(4)
|
|
QP_SessionMode=True
|
|
QP_SessionToken="''' + session + '''"
|
|
QP_ProgressSequence=0
|
|
QP_NativeMutationPossible=False
|
|
QP_RestoreTransactionOpen=False
|
|
QP_NativeCallActive=False
|
|
QP_OwnedProject=False
|
|
QP_OwnedProjectPath=""
|
|
QPProgress "session_startup",0,0
|
|
WScript.Quit 99
|
|
''' + progress + '\n' + reader, encoding='utf-8')
|
|
api, stop = _Win32Api(), Event()
|
|
stop.set()
|
|
def start(command, **options):
|
|
(tmp_path / ('monitor-' + command[-1] + '.stop')).write_text(command[-1])
|
|
return api.create_process(command, None, tmp_path, None, **options)
|
|
command = (r'C:\Windows\SysWOW64\cscript.exe', '//nologo', str(script), str(tmp_path), sys.executable, '', 'job')
|
|
try:
|
|
completed = run_monitored_process(command, 5, abort_event=stop, process_factory=start)
|
|
assert completed.returncode == 74
|
|
assert completed.startup_pause['session_token'] == session
|
|
assert completed.startup_pause['job_id'] == 'job'
|
|
assert completed.startup_pause['worker_tree_exited'] is True
|
|
finally:
|
|
api.close_pending()
|
|
|
|
|
|
@pytest.mark.skipif(os.name != 'nt', reason='Actual owned native protocol actor')
|
|
@pytest.mark.parametrize('damage', ['token', 'session', 'job', 'no_pause', 'unsafe', 'unsafe_after',
|
|
'no_progress', 'no_ack', 'malformed', 'request_changed'])
|
|
def test_monitor_rejects_unbound_or_unsafe_startup_ack(tmp_path, damage):
|
|
from dx_productivity.trial_task_center.worker_monitor import run_monitored_process, WorkerMonitorError
|
|
from dx_productivity.trial_task_center.isolated_desktop import _Win32Api
|
|
project = tmp_path / 'board.prj'
|
|
project.write_text('not opened')
|
|
request = tmp_path / 'project-session.request'
|
|
request.write_text(f'{"a"*32}\t{project}\tjob', encoding='utf-8')
|
|
script = tmp_path / 'protocol.py'
|
|
script.write_text('''import pathlib,sys,time
|
|
root=pathlib.Path(sys.argv[1]);job=sys.argv[4];token=sys.argv[5]
|
|
damage=''' + repr(damage) + '''
|
|
while not (root/f'monitor-{token}.ready').exists():time.sleep(.002)
|
|
if damage!='no_pause':
|
|
while not (root/f'monitor-{token}.stop').exists():time.sleep(.002)
|
|
if damage!='no_progress':
|
|
stage='native_replace' if damage=='unsafe' else 'session_startup'
|
|
safe=0 if damage=='unsafe' else 1
|
|
print(f'DX_PROGRESS\\t{token}\\t{job}\\t1\\t{stage}\\t0\\t0\\t{safe}',flush=True)
|
|
session='a'*32
|
|
if damage=='request_changed':
|
|
session='c'*32
|
|
p=root/'project-session.request';p.write_text(p.read_text().replace('a'*32,session))
|
|
if damage!='no_ack':
|
|
print('\\t'.join(['DX_STARTUP_PAUSED','foreign' if damage=='token' else token,job,
|
|
'd'*32 if damage=='session' else session,'foreign' if damage=='job' else job,
|
|
'invalid' if damage=='malformed' else '1']),flush=True)
|
|
if damage=='unsafe_after':print(f'DX_PROGRESS\\t{token}\\t{job}\\t2\\tnative_replace\\t0\\t1\\t0',flush=True)
|
|
sys.exit(74)
|
|
''', encoding='utf-8')
|
|
stop, api = Event(), _Win32Api()
|
|
if damage != 'no_pause': stop.set()
|
|
command = (sys.executable, '-u', str(script), str(tmp_path), sys.executable, '', 'job')
|
|
try:
|
|
with pytest.raises(WorkerMonitorError):
|
|
run_monitored_process(command, 5, abort_event=stop,
|
|
process_factory=lambda launch, **options: api.create_process(launch, None, tmp_path, None, **options))
|
|
finally:
|
|
api.close_pending()
|
|
|
|
|
|
@pytest.mark.parametrize('damage', [None, 'no_ack', 'job', 'source', 'status', 'request', 'current', 'native_started', 'checkpoint'])
|
|
def test_runtime_startup_pause_requires_prepared_attempt_and_empty_owned_project(tmp_path, monkeypatch, damage):
|
|
import subprocess
|
|
from test_batch_runtime import command_for, worker_runtime, Cad
|
|
from dx_productivity.trial_task_center import batch_attempt
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
|
command, project = command_for(tmp_path)
|
|
root = Path(command[3])
|
|
(root / 'project-session.request').write_text(f'{"a"*32}\t{project}\tjob')
|
|
def monitor(*args, **kwargs):
|
|
runtime.stop_event.set()
|
|
result = subprocess.CompletedProcess(command, 74, '', '')
|
|
if damage != 'no_ack':
|
|
result.startup_pause = {'session_token': 'a' * 32, 'monitor_token': 'b' * 32,
|
|
'job_id': 'foreign' if damage == 'job' else 'job', 'project_path': str(project),
|
|
'sequence': 1, 'worker_tree_exited': True}
|
|
if damage == 'source':
|
|
project.write_text('unexplained source mutation')
|
|
if damage == 'status':
|
|
(root / 'status.tsv').write_text('unexplained status')
|
|
if damage == 'request':
|
|
(root / 'project-session.request').write_text('changed session request')
|
|
if damage == 'current':
|
|
(root / 'current-command.tsv').write_text('changed command')
|
|
if damage == 'native_started':
|
|
path, record = runtime._current_attempts[0]
|
|
batch_attempt._publish(path, dict(record, state='mutation_possible'))
|
|
if damage == 'checkpoint':
|
|
runtime.cad.checkpoint = lambda *args: (_ for _ in ()).throw(WorkerMonitorError('project still open'))
|
|
return result
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
runtime.cad_factory = lambda *args, stop_control=None, **options: Cad(*args, **options)
|
|
runtime.stop_event = StopControl()
|
|
with runtime:
|
|
if damage:
|
|
with pytest.raises(WorkerMonitorError):
|
|
runtime.run_worker(command, 5)
|
|
else:
|
|
result = runtime.run_worker(command, 5)
|
|
assert result.returncode == 74
|
|
assert result.startup_pause['project_closed'] is True
|
|
assert result.startup_pause['native_dispatch_not_started'] is True
|
|
assert result.startup_pause['files']
|
|
assert not runtime.record['retries']
|
|
|
|
|
|
@pytest.mark.parametrize('marker_cut', [False, True])
|
|
def test_saved_uid_then_startup_pause_resumes_same_run_without_replay(component_case, tmp_path, monkeypatch, marker_cut):
|
|
import copy, json, subprocess
|
|
from test_batch_runtime import worker_runtime, Cad
|
|
from dx_productivity.dsh import execution_runs
|
|
from dx_productivity.dsh.run_journal import RunJournal
|
|
from dx_productivity.dsh.task_lifecycle import preflight
|
|
from dx_productivity.trial_task_center import native_netlist, task_queue
|
|
from dx_productivity.trial_task_center.quick_run import _decode_vbs_field
|
|
from dx_productivity.trial_task_center.stop_control import StopControl
|
|
service, store, tasks, calls, _ = component_case
|
|
task_id = tasks[0].task_id
|
|
child = service.for_project_task(task_id)
|
|
original_worker = child._quick_run_preparer._process_runner
|
|
graph = {'components': {f'u{i}': {'refdes': f'R{i}', 'attributes': {'Part Number': 'OLD', 'Value': '10K'}}
|
|
for i in (1, 2)}, 'nets': [{'name': 'GND', 'endpoints': [['u1', '1'], ['u2', '1']]}]}
|
|
def capture(project, directory):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
value = {'project': str(project), 'root': str(directory), 'graph': copy.deepcopy(graph)}
|
|
(directory / 'capture.json').write_text(json.dumps(value), encoding='utf-8')
|
|
return value
|
|
monkeypatch.setattr(native_netlist, 'capture_project', capture)
|
|
monkeypatch.setattr(native_netlist, 'export_netlist', lambda saved, *args, **kwargs:
|
|
{'status': 'success', 'complete': True, 'graph': saved['graph']})
|
|
current = {'paused': False}
|
|
def monitor(command, *args, **options):
|
|
current['runtime'].stop_event.set()
|
|
result = subprocess.CompletedProcess(command, 74, '', '')
|
|
result.startup_pause = {'monitor_token': 'b' * 32, 'session_token': 'a' * 32,
|
|
'job_id': command[-1], 'project_path': current['project'], 'sequence': 1, 'worker_tree_exited': True}
|
|
return result
|
|
def worker(command, timeout):
|
|
root = Path(command[3])
|
|
import csv
|
|
with (root / 'current-command.tsv').open(encoding='utf-8-sig') as stream:
|
|
row = {key: _decode_vbs_field(value) for key, value in next(csv.DictReader(stream, delimiter='\t')).items()}
|
|
uid = row['target_selectors'].split('\x1f')[0]
|
|
if uid == 'u2' and not current['paused']:
|
|
current['paused'] = True
|
|
current['project'] = row['project_path']
|
|
(root / 'project-session.request').write_text(f'{"a"*32}\t{row["project_path"]}\t{command[-1]}', encoding='utf-8')
|
|
return current['runtime'].run_worker(command, timeout)
|
|
result = original_worker(command, timeout)
|
|
graph['components'][uid]['attributes']['Part Number'] = 'NEW'
|
|
return result
|
|
child._quick_run_preparer._process_runner = worker
|
|
execute_queue = task_queue.execute_task_queue
|
|
def queue_with_owned_boundary(*args, stop_event, **options):
|
|
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
|
runtime.cad_factory = lambda *args, stop_control=None, **kwargs: Cad(*args, **kwargs)
|
|
runtime.stop_event = stop_event
|
|
current['runtime'] = runtime
|
|
with runtime:
|
|
return execute_queue(*args, stop_event=stop_event, runtime=runtime, **options)
|
|
monkeypatch.setattr(execution_runs, 'execute_task_queue', queue_with_owned_boundary)
|
|
data = service._logging.root
|
|
if marker_cut:
|
|
unlink = Path.unlink
|
|
def cut(path, *args, **kwargs):
|
|
if path.name == 'current-command.tsv' and not current.get('allow_finalize'):
|
|
current['cut'] = True
|
|
raise PermissionError('startup marker retirement cut')
|
|
return unlink(path, *args, **kwargs)
|
|
monkeypatch.setattr(Path, 'unlink', cut)
|
|
run_id = 'd' * 32
|
|
first = execution_runs.execute_confirmed_run(service, data, [task_id], preflight(service, [task_id]), StopControl(), run_id)
|
|
assert first['status'] == 'paused'
|
|
if marker_cut:
|
|
assert (child._quick_run_preparer.data_root / 'quick-run' / 'current-command.tsv').exists()
|
|
current['allow_finalize'] = True
|
|
assert calls == [(task_id, ('R1',))]
|
|
assert not list(data.rglob('uncertain.flag'))
|
|
rows = store._connection.execute('SELECT state FROM execution_operations WHERE task_id=? ORDER BY rowid', (task_id,)).fetchall()
|
|
assert [row[0] for row in rows] == ['verified', 'pending']
|
|
pending = store._connection.execute("SELECT * FROM execution_operations WHERE task_id=? AND state='pending'", (task_id,)).fetchone()
|
|
journal = child._quick_run_preparer.journal
|
|
assert pending['generation'] == 0
|
|
assert journal.certified_startup_pause(task_id, pending)
|
|
assert not journal.certified_startup_pause(task_id, dict(pending, generation=1))
|
|
assert not service.execution_problems(task_id)
|
|
before = data / 'tasks' / task_id / 'netlist-checks' / run_id / 'before' / 'capture.json'
|
|
before_bytes = before.read_bytes()
|
|
inspected = RunJournal(data).inspect(service, run_id)
|
|
assert inspected['resume_ready'], inspected['blockers']
|
|
resumed = execution_runs.execute_confirmed_run(service, data, [task_id],
|
|
preflight(service, [task_id], allow_completed=True), StopControl(), run_id, resume=inspected)
|
|
assert resumed['status'] == 'completed' and resumed['queue_id'] == run_id
|
|
assert calls == [(task_id, ('R1',)), (task_id, ('R2',))]
|
|
assert before.read_bytes() == before_bytes
|
|
assert not service.execution_problems(task_id)
|