feat: control and recover independent verification attempts
This commit is contained in:
1 parent
e6e9b6906d
commit
b9712ed651
7 files changed
+239
-1
No files matched your search
@@ -358,6 +358,12 @@ class TrialTaskCenterWindow(MaterialMainWindow):
|
||||
if self._close_saving:
|
||||
event.ignore()
|
||||
return
|
||||
if self.verification_panel.busy:
|
||||
self._close_pending = True
|
||||
self.verification_panel.begin_shutdown()
|
||||
event.ignore()
|
||||
QTimer.singleShot(100, self.close)
|
||||
return
|
||||
if self._workers or self.history_panel._worker:
|
||||
self._close_pending = True
|
||||
if self._dsh_automation and getattr(self._dsh_automation, '_owns_execution_lane', False):
|
||||
|
||||
@@ -68,7 +68,7 @@ class VerificationWorkspace(QWidget):
|
||||
def set_active(self,active):
|
||||
self._active=active
|
||||
if active:
|
||||
self._timer.start();self.controller.command(self.service.reconcile_deliveries,lambda _:self.refresh())
|
||||
self._timer.start();self.controller.command(self.service.initialize,lambda _:self.refresh())
|
||||
else:self._timer.stop()
|
||||
|
||||
def refresh(self):
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Frozen run membership and cooperative cancellation for the verification domain."""
|
||||
import json
|
||||
from threading import Event,Thread,RLock
|
||||
from uuid import uuid4
|
||||
from .contracts import ACTIVE,digest,fail
|
||||
from .store import encode,now
|
||||
|
||||
|
||||
class VerificationControl:
|
||||
def __init__(self,service):
|
||||
self.service=service;self.repo=service.repo;self.lock=RLock();self.events={};self.thread=None;self.closing=False
|
||||
|
||||
@property
|
||||
def busy(self):
|
||||
with self.lock:return bool(self.thread and self.thread.is_alive())
|
||||
|
||||
def start(self,task_id,*,selection,request_id):
|
||||
if self.closing:fail('校验执行器正在安全关闭。')
|
||||
body=[task_id,selection]
|
||||
with self.repo.store.transaction():
|
||||
old=self.repo._request('start',request_id,body)
|
||||
if old is not None:return old
|
||||
self.repo.task(task_id)
|
||||
active=self.repo.connection.execute("SELECT * FROM verification_runs WHERE task_id=? AND state IN ('queued','running','stopping') ORDER BY created LIMIT 1",(task_id,)).fetchone()
|
||||
if active:
|
||||
result={'run_id':active['run_id'],'members':json.loads(active['members_json'])}
|
||||
else:
|
||||
rows=self.service.queries._rows(task_id)
|
||||
wanted=set(tuple(v) for v in selection) if selection is not None else None
|
||||
members=[];run_id=uuid4().hex
|
||||
for row in rows:
|
||||
identity=(row['item_id'],row['input_version'])
|
||||
if wanted is not None and identity not in wanted:continue
|
||||
if not row['can_start']:continue
|
||||
# Start all only processes waiting/interrupted items; explicit selection permits recheck.
|
||||
if wanted is None and row['status'] not in {'waiting','interrupted'}:continue
|
||||
attempt=self.repo.create_attempt(*identity,request_id='run-attempt:'+run_id+':'+row['item_id'],run_id=run_id)
|
||||
members.append({'item_id':identity[0],'input_version':identity[1],'attempt_id':attempt['attempt_id']})
|
||||
if wanted and wanted-{(m['item_id'],m['input_version']) for m in members}:fail('所选版本已变化或正在执行,请刷新。')
|
||||
self.repo.connection.execute('INSERT INTO verification_runs(run_id,task_id,members_json,state,created) VALUES (?,?,?,?,?)',
|
||||
(run_id,task_id,encode(members),'queued' if members else 'completed',now()))
|
||||
result={'run_id':run_id,'members':members}
|
||||
self.repo._remember('start',request_id,body,result)
|
||||
self.wake()
|
||||
return result
|
||||
|
||||
def wake(self):
|
||||
with self.lock:
|
||||
if not self.closing and not self.busy:
|
||||
self.thread=Thread(target=self._dispatch,name='verification-runner',daemon=True);self.thread.start()
|
||||
|
||||
def stop(self,run_id,*,request_id):
|
||||
with self.repo.store.transaction():
|
||||
old=self.repo._request('stop',request_id,run_id)
|
||||
if old is not None:return old
|
||||
run=self.repo._one('verification_runs','run_id',run_id)
|
||||
self.repo.connection.execute("UPDATE verification_runs SET stop_requested=1,state=CASE WHEN state IN ('queued','running') THEN 'stopping' ELSE state END WHERE run_id=?",(run_id,))
|
||||
for member in json.loads(run['members_json']):
|
||||
self.repo.connection.execute("UPDATE verification_attempts SET state='cancelled',reason='run_stopped_before_dispatch',resource_state='released',updated=? WHERE attempt_id=? AND state='queued'",(now(),member['attempt_id']))
|
||||
self.repo._touch(run['task_id']);result={'run_id':run_id,'stop_requested':True}
|
||||
self.repo._remember('stop',request_id,run_id,result)
|
||||
with self.lock:self.events.setdefault(run_id,Event()).set()
|
||||
return result
|
||||
|
||||
def _dispatch(self):
|
||||
from .runner import run_attempt
|
||||
try:
|
||||
while not self.closing:
|
||||
with self.repo.store.transaction():
|
||||
run=self.repo.connection.execute("SELECT * FROM verification_runs WHERE state IN ('queued','running','stopping') ORDER BY created LIMIT 1").fetchone()
|
||||
if run is None:return
|
||||
run=dict(run)
|
||||
if not run['stop_requested']:
|
||||
self.repo.connection.execute("UPDATE verification_runs SET state='running' WHERE run_id=?",(run['run_id'],))
|
||||
with self.lock:event=self.events.setdefault(run['run_id'],Event())
|
||||
if run['stop_requested']:event.set()
|
||||
blocked=False
|
||||
for member in json.loads(run['members_json']):
|
||||
result={}
|
||||
if event.is_set() or self.closing:break
|
||||
while not event.is_set() and not self.closing:
|
||||
result=run_attempt(self.service,member['attempt_id'],stop_event=event)
|
||||
if not result.get('waiting'):break
|
||||
event.wait(.2)
|
||||
if result.get('blocked'):blocked=True;break
|
||||
with self.repo.store.transaction():
|
||||
state='blocked' if blocked else 'stopped' if event.is_set() or self.closing else 'completed'
|
||||
self.repo.connection.execute('UPDATE verification_runs SET state=? WHERE run_id=?',(state,run['run_id']))
|
||||
self.repo._touch(run['task_id'])
|
||||
finally:
|
||||
with self.lock:
|
||||
self.thread=None
|
||||
# Close the enqueue/empty-dispatch race without polling from the UI.
|
||||
if not self.closing:
|
||||
with self.repo.store._lock:
|
||||
pending=self.repo.connection.execute("SELECT 1 FROM verification_runs WHERE state IN ('queued','running','stopping') LIMIT 1").fetchone()
|
||||
if pending:self.wake()
|
||||
|
||||
def recover(self):
|
||||
if self.busy:return {'interrupted':0,'blocked':0,'published':0}
|
||||
result={'interrupted':0,'blocked':0,'published':0}
|
||||
with self.repo.store.transaction():
|
||||
rows=self.repo.connection.execute("SELECT * FROM verification_attempts WHERE state IN ('queued','running','publishing','stopping')").fetchall()
|
||||
for raw in rows:
|
||||
row=dict(raw);identity=row['attempt_id'];directory=self.service.data_root/'verification'/'attempts'/identity
|
||||
if row['state']=='queued':state,resource='interrupted','released'
|
||||
else:
|
||||
try:
|
||||
ready=json.loads((directory/'result-ready.json').read_text(encoding='utf-8'))
|
||||
value=self.service.artifacts.read(ready['result_ref'],'result')
|
||||
if value['attempt_id']!=identity or value['generation']!=row['generation']:raise ValueError('身份不符')
|
||||
from .report import validate_result
|
||||
validate_result(value,self.service.data_root)
|
||||
if row['state'] in {'running','publishing'} and self.repo.finish(identity,generation=row['generation'],result=dict(value,result_ref=ready['result_ref'])):
|
||||
result['published']+=1;continue
|
||||
except (OSError,ValueError,KeyError,ImportError):pass
|
||||
try:
|
||||
proof=json.loads((directory/'resource-released.json').read_text(encoding='utf-8'))
|
||||
if proof['owner_id']!=row['owner_id'] or proof['resource_state']!='released':raise ValueError('退出未知')
|
||||
state,resource='interrupted','released'
|
||||
except (OSError,ValueError,KeyError):state,resource='stopping','unknown'
|
||||
self.repo.connection.execute('UPDATE verification_attempts SET state=?,resource_state=?,reason=?,updated=? WHERE attempt_id=?',
|
||||
(state,resource,'上次执行中断;未自动重跑' if resource=='released' else '上次原生进程退出缺少证明,保留现场',now(),identity))
|
||||
self.repo._touch(self.repo.item(row['item_id'])['task_id'])
|
||||
result['blocked' if resource=='unknown' else 'interrupted']+=1
|
||||
self.repo.connection.execute("UPDATE verification_runs SET state='interrupted' WHERE state IN ('queued','running','stopping')")
|
||||
return result
|
||||
|
||||
def begin_shutdown(self):
|
||||
self.closing=True
|
||||
with self.repo.store._lock:
|
||||
runs=[r[0] for r in self.repo.connection.execute("SELECT run_id FROM verification_runs WHERE state IN ('queued','running','stopping')")]
|
||||
for identity in runs:self.stop(identity,request_id='shutdown:'+identity)
|
||||
with self.lock:
|
||||
for event in self.events.values():event.set()
|
||||
@@ -43,6 +43,8 @@ def run_attempt(service,attempt_id,*,stop_event):
|
||||
with service.repo.store.transaction():
|
||||
service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",('原生资源退出未知',attempt_id))
|
||||
return {'published':False,'blocked':True}
|
||||
from ..batch_resources import write_record
|
||||
write_record(output/'resource-released.json',{'owner_id':attempt['owner_id'],'resource_state':'released'})
|
||||
check_cancelled(stop_event)
|
||||
expected=service.artifacts.read(payload['expectation_ref'],'expectations') if payload['expectation_ref'] else None
|
||||
from .rules import evaluate
|
||||
@@ -66,11 +68,17 @@ def run_attempt(service,attempt_id,*,stop_event):
|
||||
'input_digest':payload['payload_digest'],'raw':raw})
|
||||
ref=service.artifacts.put('result',result)
|
||||
result['result_ref']=ref
|
||||
write_record(output/'result-ready.json',{'result_ref':ref})
|
||||
check_cancelled(stop_event)
|
||||
published=service.repo.finish(attempt_id,generation=attempt['generation'],result=result)
|
||||
return dict(result,published=published)
|
||||
except Exception as error:
|
||||
if attempt:
|
||||
from ..native_cli import _UNCERTAIN_SCOPES
|
||||
if _UNCERTAIN_SCOPES:
|
||||
with service.repo.store.transaction():
|
||||
service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",(str(error),attempt_id))
|
||||
return {'published':False,'blocked':True}
|
||||
cancelled=getattr(error,'code','')=='operation_cancelled'
|
||||
result={'attempt_state':'cancelled' if cancelled else 'failed','resource_state':'released',
|
||||
'verdict':'INCOMPLETE','reason':str(error),'checks':[],'issues':[]}
|
||||
|
||||
@@ -19,6 +19,28 @@ class VerificationService:
|
||||
self._intake_lock = RLock()
|
||||
from .queries import VerificationQueries
|
||||
self.queries = VerificationQueries(self.repo)
|
||||
from .control import VerificationControl
|
||||
self.control = VerificationControl(self)
|
||||
self._recovered=False
|
||||
|
||||
def initialize(self):
|
||||
with self._intake_lock:
|
||||
if not self._recovered:
|
||||
self.recover()
|
||||
self._recovered=True
|
||||
return self.reconcile_deliveries()
|
||||
|
||||
@property
|
||||
def busy(self): return self.control.busy
|
||||
|
||||
def start(self, task_id, *, selection=None, request_id):
|
||||
return self.control.start(task_id,selection=selection,request_id=request_id)
|
||||
|
||||
def stop(self, run_id, *, request_id): return self.control.stop(run_id,request_id=request_id)
|
||||
def rerun(self, item_id, version, *, request_id):
|
||||
return self.start(self.repo.item(item_id)['task_id'],selection=((item_id,version),),request_id=request_id)
|
||||
def recover(self): return self.control.recover()
|
||||
def begin_shutdown(self): return self.control.begin_shutdown()
|
||||
|
||||
def list_tasks(self, **kwargs): return self.queries.list_tasks(**kwargs)
|
||||
def read_task(self, task_id, **kwargs): return self.queries.read_task(task_id, **kwargs)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
def test_close_and_controls_do_not_block_new_task_creation(ui,qtbot):
|
||||
page=ui.window.verification_panel
|
||||
task=ui.service.verification.create_task('任务',request_id='control')['task_id']
|
||||
page.open_task(task)
|
||||
qtbot.waitUntil(lambda:not page.controller.task_query.busy)
|
||||
assert page.start_button.isEnabled() and page.new_button.isEnabled()
|
||||
page.begin_shutdown()
|
||||
assert page.new_button.isEnabled()
|
||||
@@ -0,0 +1,59 @@
|
||||
import time
|
||||
from threading import Event
|
||||
from test_independent_verification_intake import service
|
||||
from test_independent_verification_runner import imported
|
||||
from test_native_cli_netlist import graph
|
||||
|
||||
|
||||
def wait_idle(service):
|
||||
deadline=time.monotonic()+5
|
||||
while service.busy and time.monotonic()<deadline:time.sleep(.01)
|
||||
assert not service.busy
|
||||
|
||||
|
||||
def test_start_idempotency_and_frozen_members_do_not_swallow_new_version(service,tmp_path,monkeypatch):
|
||||
from dx_productivity.trial_task_center import netlist_verification
|
||||
entered=Event();release=Event();calls=[]
|
||||
def export(capture,output,*,stop_event):
|
||||
calls.append(str(output));entered.set();release.wait(3);output.mkdir(parents=True,exist_ok=True)
|
||||
return {'status':'success','complete':True,'graph':graph(output),'outputs':{}}
|
||||
monkeypatch.setattr(netlist_verification,'_export_saved_capture',export)
|
||||
task,item=imported(service,tmp_path)
|
||||
first=service.start(task,selection=None,request_id='run')
|
||||
assert entered.wait(2)
|
||||
assert service.start(task,selection=None,request_id='run')==first
|
||||
payload=service.repo.revision(item,1)['payload']
|
||||
newer={k:v for k,v in payload.items() if k not in {'input_state','payload_digest'}};newer['input_version']=2
|
||||
service.publish_inputs(task,(newer,),submission_id='new')
|
||||
release.set();wait_idle(service)
|
||||
assert len(calls)==2 and service.read_item(item)['status']=='waiting'
|
||||
assert service.read_item(item,version=1)['status']=='consistent'
|
||||
|
||||
|
||||
def test_stop_during_export_is_cancelled_and_rerun_is_new_generation(service,tmp_path,monkeypatch):
|
||||
from dx_productivity.trial_task_center import netlist_verification
|
||||
entered=Event()
|
||||
def slow(capture,output,*,stop_event):
|
||||
entered.set();stop_event.wait(3)
|
||||
return {'status':'cancelled','complete':False,'reason':'stopped'}
|
||||
monkeypatch.setattr(netlist_verification,'_export_saved_capture',slow)
|
||||
task,item=imported(service,tmp_path)
|
||||
run=service.start(task,selection=None,request_id='start');assert entered.wait(2)
|
||||
service.stop(run['run_id'],request_id='stop');wait_idle(service)
|
||||
assert service.read_item(item)['status']=='stopped'
|
||||
def normal(capture,output,*,stop_event):
|
||||
output.mkdir(parents=True,exist_ok=True)
|
||||
return {'status':'success','complete':True,'graph':graph(output),'outputs':{}}
|
||||
monkeypatch.setattr(netlist_verification,'_export_saved_capture',normal)
|
||||
service.rerun(item,1,request_id='again');wait_idle(service)
|
||||
assert service.read_item(item)['attempt']['generation']==2
|
||||
|
||||
|
||||
def test_recovery_never_automatically_runs_and_unknown_owner_blocks(service,tmp_path):
|
||||
task,item=imported(service,tmp_path)
|
||||
attempt=service.repo.create_attempt(item,1,request_id='lost')
|
||||
service.repo.claim(attempt['attempt_id'],owner_id='lost-owner')
|
||||
result=service.recover()
|
||||
assert result['blocked']==1 and not service.busy
|
||||
assert service.repo.attempt(attempt['attempt_id'])['state']=='stopping'
|
||||
assert service.read_item(item)['verdict'] is None
|
||||
Reference in new issue
Block a user