feat: control and recover independent verification attempts

This commit is contained in:
阿白叔 committed 2026-11-17 15:59:28 +08:00
1 parent e6e9b6906d
commit b9712ed651
7 files changed
+239 -1

No files matched your search

@@ -358,6 +358,12 @@ class TrialTaskCenterWindow(MaterialMainWindow):
if self._close_saving:
event.ignore()
return
if self.verification_panel.busy:
self._close_pending = True
self.verification_panel.begin_shutdown()
event.ignore()
QTimer.singleShot(100, self.close)
return
if self._workers or self.history_panel._worker:
self._close_pending = True
if self._dsh_automation and getattr(self._dsh_automation, '_owns_execution_lane', False):
@@ -68,7 +68,7 @@ class VerificationWorkspace(QWidget):
def set_active(self,active):
self._active=active
if active:
self._timer.start();self.controller.command(self.service.reconcile_deliveries,lambda _:self.refresh())
self._timer.start();self.controller.command(self.service.initialize,lambda _:self.refresh())
else:self._timer.stop()
def refresh(self):
@@ -0,0 +1,135 @@
"""Frozen run membership and cooperative cancellation for the verification domain."""
import json
from threading import Event,Thread,RLock
from uuid import uuid4
from .contracts import ACTIVE,digest,fail
from .store import encode,now
class VerificationControl:
def __init__(self,service):
self.service=service;self.repo=service.repo;self.lock=RLock();self.events={};self.thread=None;self.closing=False
@property
def busy(self):
with self.lock:return bool(self.thread and self.thread.is_alive())
def start(self,task_id,*,selection,request_id):
if self.closing:fail('校验执行器正在安全关闭。')
body=[task_id,selection]
with self.repo.store.transaction():
old=self.repo._request('start',request_id,body)
if old is not None:return old
self.repo.task(task_id)
active=self.repo.connection.execute("SELECT * FROM verification_runs WHERE task_id=? AND state IN ('queued','running','stopping') ORDER BY created LIMIT 1",(task_id,)).fetchone()
if active:
result={'run_id':active['run_id'],'members':json.loads(active['members_json'])}
else:
rows=self.service.queries._rows(task_id)
wanted=set(tuple(v) for v in selection) if selection is not None else None
members=[];run_id=uuid4().hex
for row in rows:
identity=(row['item_id'],row['input_version'])
if wanted is not None and identity not in wanted:continue
if not row['can_start']:continue
# Start all only processes waiting/interrupted items; explicit selection permits recheck.
if wanted is None and row['status'] not in {'waiting','interrupted'}:continue
attempt=self.repo.create_attempt(*identity,request_id='run-attempt:'+run_id+':'+row['item_id'],run_id=run_id)
members.append({'item_id':identity[0],'input_version':identity[1],'attempt_id':attempt['attempt_id']})
if wanted and wanted-{(m['item_id'],m['input_version']) for m in members}:fail('所选版本已变化或正在执行,请刷新。')
self.repo.connection.execute('INSERT INTO verification_runs(run_id,task_id,members_json,state,created) VALUES (?,?,?,?,?)',
(run_id,task_id,encode(members),'queued' if members else 'completed',now()))
result={'run_id':run_id,'members':members}
self.repo._remember('start',request_id,body,result)
self.wake()
return result
def wake(self):
with self.lock:
if not self.closing and not self.busy:
self.thread=Thread(target=self._dispatch,name='verification-runner',daemon=True);self.thread.start()
def stop(self,run_id,*,request_id):
with self.repo.store.transaction():
old=self.repo._request('stop',request_id,run_id)
if old is not None:return old
run=self.repo._one('verification_runs','run_id',run_id)
self.repo.connection.execute("UPDATE verification_runs SET stop_requested=1,state=CASE WHEN state IN ('queued','running') THEN 'stopping' ELSE state END WHERE run_id=?",(run_id,))
for member in json.loads(run['members_json']):
self.repo.connection.execute("UPDATE verification_attempts SET state='cancelled',reason='run_stopped_before_dispatch',resource_state='released',updated=? WHERE attempt_id=? AND state='queued'",(now(),member['attempt_id']))
self.repo._touch(run['task_id']);result={'run_id':run_id,'stop_requested':True}
self.repo._remember('stop',request_id,run_id,result)
with self.lock:self.events.setdefault(run_id,Event()).set()
return result
def _dispatch(self):
from .runner import run_attempt
try:
while not self.closing:
with self.repo.store.transaction():
run=self.repo.connection.execute("SELECT * FROM verification_runs WHERE state IN ('queued','running','stopping') ORDER BY created LIMIT 1").fetchone()
if run is None:return
run=dict(run)
if not run['stop_requested']:
self.repo.connection.execute("UPDATE verification_runs SET state='running' WHERE run_id=?",(run['run_id'],))
with self.lock:event=self.events.setdefault(run['run_id'],Event())
if run['stop_requested']:event.set()
blocked=False
for member in json.loads(run['members_json']):
result={}
if event.is_set() or self.closing:break
while not event.is_set() and not self.closing:
result=run_attempt(self.service,member['attempt_id'],stop_event=event)
if not result.get('waiting'):break
event.wait(.2)
if result.get('blocked'):blocked=True;break
with self.repo.store.transaction():
state='blocked' if blocked else 'stopped' if event.is_set() or self.closing else 'completed'
self.repo.connection.execute('UPDATE verification_runs SET state=? WHERE run_id=?',(state,run['run_id']))
self.repo._touch(run['task_id'])
finally:
with self.lock:
self.thread=None
# Close the enqueue/empty-dispatch race without polling from the UI.
if not self.closing:
with self.repo.store._lock:
pending=self.repo.connection.execute("SELECT 1 FROM verification_runs WHERE state IN ('queued','running','stopping') LIMIT 1").fetchone()
if pending:self.wake()
def recover(self):
if self.busy:return {'interrupted':0,'blocked':0,'published':0}
result={'interrupted':0,'blocked':0,'published':0}
with self.repo.store.transaction():
rows=self.repo.connection.execute("SELECT * FROM verification_attempts WHERE state IN ('queued','running','publishing','stopping')").fetchall()
for raw in rows:
row=dict(raw);identity=row['attempt_id'];directory=self.service.data_root/'verification'/'attempts'/identity
if row['state']=='queued':state,resource='interrupted','released'
else:
try:
ready=json.loads((directory/'result-ready.json').read_text(encoding='utf-8'))
value=self.service.artifacts.read(ready['result_ref'],'result')
if value['attempt_id']!=identity or value['generation']!=row['generation']:raise ValueError('身份不符')
from .report import validate_result
validate_result(value,self.service.data_root)
if row['state'] in {'running','publishing'} and self.repo.finish(identity,generation=row['generation'],result=dict(value,result_ref=ready['result_ref'])):
result['published']+=1;continue
except (OSError,ValueError,KeyError,ImportError):pass
try:
proof=json.loads((directory/'resource-released.json').read_text(encoding='utf-8'))
if proof['owner_id']!=row['owner_id'] or proof['resource_state']!='released':raise ValueError('退出未知')
state,resource='interrupted','released'
except (OSError,ValueError,KeyError):state,resource='stopping','unknown'
self.repo.connection.execute('UPDATE verification_attempts SET state=?,resource_state=?,reason=?,updated=? WHERE attempt_id=?',
(state,resource,'上次执行中断;未自动重跑' if resource=='released' else '上次原生进程退出缺少证明,保留现场',now(),identity))
self.repo._touch(self.repo.item(row['item_id'])['task_id'])
result['blocked' if resource=='unknown' else 'interrupted']+=1
self.repo.connection.execute("UPDATE verification_runs SET state='interrupted' WHERE state IN ('queued','running','stopping')")
return result
def begin_shutdown(self):
self.closing=True
with self.repo.store._lock:
runs=[r[0] for r in self.repo.connection.execute("SELECT run_id FROM verification_runs WHERE state IN ('queued','running','stopping')")]
for identity in runs:self.stop(identity,request_id='shutdown:'+identity)
with self.lock:
for event in self.events.values():event.set()
@@ -43,6 +43,8 @@ def run_attempt(service,attempt_id,*,stop_event):
with service.repo.store.transaction():
service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",('原生资源退出未知',attempt_id))
return {'published':False,'blocked':True}
from ..batch_resources import write_record
write_record(output/'resource-released.json',{'owner_id':attempt['owner_id'],'resource_state':'released'})
check_cancelled(stop_event)
expected=service.artifacts.read(payload['expectation_ref'],'expectations') if payload['expectation_ref'] else None
from .rules import evaluate
@@ -66,11 +68,17 @@ def run_attempt(service,attempt_id,*,stop_event):
'input_digest':payload['payload_digest'],'raw':raw})
ref=service.artifacts.put('result',result)
result['result_ref']=ref
write_record(output/'result-ready.json',{'result_ref':ref})
check_cancelled(stop_event)
published=service.repo.finish(attempt_id,generation=attempt['generation'],result=result)
return dict(result,published=published)
except Exception as error:
if attempt:
from ..native_cli import _UNCERTAIN_SCOPES
if _UNCERTAIN_SCOPES:
with service.repo.store.transaction():
service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",(str(error),attempt_id))
return {'published':False,'blocked':True}
cancelled=getattr(error,'code','')=='operation_cancelled'
result={'attempt_state':'cancelled' if cancelled else 'failed','resource_state':'released',
'verdict':'INCOMPLETE','reason':str(error),'checks':[],'issues':[]}
@@ -19,6 +19,28 @@ class VerificationService:
self._intake_lock = RLock()
from .queries import VerificationQueries
self.queries = VerificationQueries(self.repo)
from .control import VerificationControl
self.control = VerificationControl(self)
self._recovered=False
def initialize(self):
with self._intake_lock:
if not self._recovered:
self.recover()
self._recovered=True
return self.reconcile_deliveries()
@property
def busy(self): return self.control.busy
def start(self, task_id, *, selection=None, request_id):
return self.control.start(task_id,selection=selection,request_id=request_id)
def stop(self, run_id, *, request_id): return self.control.stop(run_id,request_id=request_id)
def rerun(self, item_id, version, *, request_id):
return self.start(self.repo.item(item_id)['task_id'],selection=((item_id,version),),request_id=request_id)
def recover(self): return self.control.recover()
def begin_shutdown(self): return self.control.begin_shutdown()
def list_tasks(self, **kwargs): return self.queries.list_tasks(**kwargs)
def read_task(self, task_id, **kwargs): return self.queries.read_task(task_id, **kwargs)
+8
View File
@@ -0,0 +1,8 @@
def test_close_and_controls_do_not_block_new_task_creation(ui,qtbot):
page=ui.window.verification_panel
task=ui.service.verification.create_task('任务',request_id='control')['task_id']
page.open_task(task)
qtbot.waitUntil(lambda:not page.controller.task_query.busy)
assert page.start_button.isEnabled() and page.new_button.isEnabled()
page.begin_shutdown()
assert page.new_button.isEnabled()
+59
View File
@@ -0,0 +1,59 @@
import time
from threading import Event
from test_independent_verification_intake import service
from test_independent_verification_runner import imported
from test_native_cli_netlist import graph
def wait_idle(service):
deadline=time.monotonic()+5
while service.busy and time.monotonic()<deadline:time.sleep(.01)
assert not service.busy
def test_start_idempotency_and_frozen_members_do_not_swallow_new_version(service,tmp_path,monkeypatch):
from dx_productivity.trial_task_center import netlist_verification
entered=Event();release=Event();calls=[]
def export(capture,output,*,stop_event):
calls.append(str(output));entered.set();release.wait(3);output.mkdir(parents=True,exist_ok=True)
return {'status':'success','complete':True,'graph':graph(output),'outputs':{}}
monkeypatch.setattr(netlist_verification,'_export_saved_capture',export)
task,item=imported(service,tmp_path)
first=service.start(task,selection=None,request_id='run')
assert entered.wait(2)
assert service.start(task,selection=None,request_id='run')==first
payload=service.repo.revision(item,1)['payload']
newer={k:v for k,v in payload.items() if k not in {'input_state','payload_digest'}};newer['input_version']=2
service.publish_inputs(task,(newer,),submission_id='new')
release.set();wait_idle(service)
assert len(calls)==2 and service.read_item(item)['status']=='waiting'
assert service.read_item(item,version=1)['status']=='consistent'
def test_stop_during_export_is_cancelled_and_rerun_is_new_generation(service,tmp_path,monkeypatch):
from dx_productivity.trial_task_center import netlist_verification
entered=Event()
def slow(capture,output,*,stop_event):
entered.set();stop_event.wait(3)
return {'status':'cancelled','complete':False,'reason':'stopped'}
monkeypatch.setattr(netlist_verification,'_export_saved_capture',slow)
task,item=imported(service,tmp_path)
run=service.start(task,selection=None,request_id='start');assert entered.wait(2)
service.stop(run['run_id'],request_id='stop');wait_idle(service)
assert service.read_item(item)['status']=='stopped'
def normal(capture,output,*,stop_event):
output.mkdir(parents=True,exist_ok=True)
return {'status':'success','complete':True,'graph':graph(output),'outputs':{}}
monkeypatch.setattr(netlist_verification,'_export_saved_capture',normal)
service.rerun(item,1,request_id='again');wait_idle(service)
assert service.read_item(item)['attempt']['generation']==2
def test_recovery_never_automatically_runs_and_unknown_owner_blocks(service,tmp_path):
task,item=imported(service,tmp_path)
attempt=service.repo.create_attempt(item,1,request_id='lost')
service.repo.claim(attempt['attempt_id'],owner_id='lost-owner')
result=service.recover()
assert result['blocked']==1 and not service.busy
assert service.repo.attempt(attempt['attempt_id'])['state']=='stopping'
assert service.read_item(item)['verdict'] is None