diff --git a/src/dx_productivity/trial_task_center/ui/main_window.py b/src/dx_productivity/trial_task_center/ui/main_window.py index 66f1fc0..aeaf736 100644 --- a/src/dx_productivity/trial_task_center/ui/main_window.py +++ b/src/dx_productivity/trial_task_center/ui/main_window.py @@ -358,6 +358,12 @@ class TrialTaskCenterWindow(MaterialMainWindow): if self._close_saving: event.ignore() return + if self.verification_panel.busy: + self._close_pending = True + self.verification_panel.begin_shutdown() + event.ignore() + QTimer.singleShot(100, self.close) + return if self._workers or self.history_panel._worker: self._close_pending = True if self._dsh_automation and getattr(self._dsh_automation, '_owns_execution_lane', False): diff --git a/src/dx_productivity/trial_task_center/ui/verification_workspace.py b/src/dx_productivity/trial_task_center/ui/verification_workspace.py index 222ac1d..7181a54 100644 --- a/src/dx_productivity/trial_task_center/ui/verification_workspace.py +++ b/src/dx_productivity/trial_task_center/ui/verification_workspace.py @@ -68,7 +68,7 @@ class VerificationWorkspace(QWidget): def set_active(self,active): self._active=active if active: - self._timer.start();self.controller.command(self.service.reconcile_deliveries,lambda _:self.refresh()) + self._timer.start();self.controller.command(self.service.initialize,lambda _:self.refresh()) else:self._timer.stop() def refresh(self): diff --git a/src/dx_productivity/trial_task_center/verification/control.py b/src/dx_productivity/trial_task_center/verification/control.py new file mode 100644 index 0000000..f3a9017 --- /dev/null +++ b/src/dx_productivity/trial_task_center/verification/control.py @@ -0,0 +1,135 @@ +"""Frozen run membership and cooperative cancellation for the verification domain.""" +import json +from threading import Event,Thread,RLock +from uuid import uuid4 +from .contracts import ACTIVE,digest,fail +from .store import encode,now + + +class VerificationControl: + def __init__(self,service): + self.service=service;self.repo=service.repo;self.lock=RLock();self.events={};self.thread=None;self.closing=False + + @property + def busy(self): + with self.lock:return bool(self.thread and self.thread.is_alive()) + + def start(self,task_id,*,selection,request_id): + if self.closing:fail('校验执行器正在安全关闭。') + body=[task_id,selection] + with self.repo.store.transaction(): + old=self.repo._request('start',request_id,body) + if old is not None:return old + self.repo.task(task_id) + active=self.repo.connection.execute("SELECT * FROM verification_runs WHERE task_id=? AND state IN ('queued','running','stopping') ORDER BY created LIMIT 1",(task_id,)).fetchone() + if active: + result={'run_id':active['run_id'],'members':json.loads(active['members_json'])} + else: + rows=self.service.queries._rows(task_id) + wanted=set(tuple(v) for v in selection) if selection is not None else None + members=[];run_id=uuid4().hex + for row in rows: + identity=(row['item_id'],row['input_version']) + if wanted is not None and identity not in wanted:continue + if not row['can_start']:continue + # Start all only processes waiting/interrupted items; explicit selection permits recheck. + if wanted is None and row['status'] not in {'waiting','interrupted'}:continue + attempt=self.repo.create_attempt(*identity,request_id='run-attempt:'+run_id+':'+row['item_id'],run_id=run_id) + members.append({'item_id':identity[0],'input_version':identity[1],'attempt_id':attempt['attempt_id']}) + if wanted and wanted-{(m['item_id'],m['input_version']) for m in members}:fail('所选版本已变化或正在执行,请刷新。') + self.repo.connection.execute('INSERT INTO verification_runs(run_id,task_id,members_json,state,created) VALUES (?,?,?,?,?)', + (run_id,task_id,encode(members),'queued' if members else 'completed',now())) + result={'run_id':run_id,'members':members} + self.repo._remember('start',request_id,body,result) + self.wake() + return result + + def wake(self): + with self.lock: + if not self.closing and not self.busy: + self.thread=Thread(target=self._dispatch,name='verification-runner',daemon=True);self.thread.start() + + def stop(self,run_id,*,request_id): + with self.repo.store.transaction(): + old=self.repo._request('stop',request_id,run_id) + if old is not None:return old + run=self.repo._one('verification_runs','run_id',run_id) + self.repo.connection.execute("UPDATE verification_runs SET stop_requested=1,state=CASE WHEN state IN ('queued','running') THEN 'stopping' ELSE state END WHERE run_id=?",(run_id,)) + for member in json.loads(run['members_json']): + self.repo.connection.execute("UPDATE verification_attempts SET state='cancelled',reason='run_stopped_before_dispatch',resource_state='released',updated=? WHERE attempt_id=? AND state='queued'",(now(),member['attempt_id'])) + self.repo._touch(run['task_id']);result={'run_id':run_id,'stop_requested':True} + self.repo._remember('stop',request_id,run_id,result) + with self.lock:self.events.setdefault(run_id,Event()).set() + return result + + def _dispatch(self): + from .runner import run_attempt + try: + while not self.closing: + with self.repo.store.transaction(): + run=self.repo.connection.execute("SELECT * FROM verification_runs WHERE state IN ('queued','running','stopping') ORDER BY created LIMIT 1").fetchone() + if run is None:return + run=dict(run) + if not run['stop_requested']: + self.repo.connection.execute("UPDATE verification_runs SET state='running' WHERE run_id=?",(run['run_id'],)) + with self.lock:event=self.events.setdefault(run['run_id'],Event()) + if run['stop_requested']:event.set() + blocked=False + for member in json.loads(run['members_json']): + result={} + if event.is_set() or self.closing:break + while not event.is_set() and not self.closing: + result=run_attempt(self.service,member['attempt_id'],stop_event=event) + if not result.get('waiting'):break + event.wait(.2) + if result.get('blocked'):blocked=True;break + with self.repo.store.transaction(): + state='blocked' if blocked else 'stopped' if event.is_set() or self.closing else 'completed' + self.repo.connection.execute('UPDATE verification_runs SET state=? WHERE run_id=?',(state,run['run_id'])) + self.repo._touch(run['task_id']) + finally: + with self.lock: + self.thread=None + # Close the enqueue/empty-dispatch race without polling from the UI. + if not self.closing: + with self.repo.store._lock: + pending=self.repo.connection.execute("SELECT 1 FROM verification_runs WHERE state IN ('queued','running','stopping') LIMIT 1").fetchone() + if pending:self.wake() + + def recover(self): + if self.busy:return {'interrupted':0,'blocked':0,'published':0} + result={'interrupted':0,'blocked':0,'published':0} + with self.repo.store.transaction(): + rows=self.repo.connection.execute("SELECT * FROM verification_attempts WHERE state IN ('queued','running','publishing','stopping')").fetchall() + for raw in rows: + row=dict(raw);identity=row['attempt_id'];directory=self.service.data_root/'verification'/'attempts'/identity + if row['state']=='queued':state,resource='interrupted','released' + else: + try: + ready=json.loads((directory/'result-ready.json').read_text(encoding='utf-8')) + value=self.service.artifacts.read(ready['result_ref'],'result') + if value['attempt_id']!=identity or value['generation']!=row['generation']:raise ValueError('身份不符') + from .report import validate_result + validate_result(value,self.service.data_root) + if row['state'] in {'running','publishing'} and self.repo.finish(identity,generation=row['generation'],result=dict(value,result_ref=ready['result_ref'])): + result['published']+=1;continue + except (OSError,ValueError,KeyError,ImportError):pass + try: + proof=json.loads((directory/'resource-released.json').read_text(encoding='utf-8')) + if proof['owner_id']!=row['owner_id'] or proof['resource_state']!='released':raise ValueError('退出未知') + state,resource='interrupted','released' + except (OSError,ValueError,KeyError):state,resource='stopping','unknown' + self.repo.connection.execute('UPDATE verification_attempts SET state=?,resource_state=?,reason=?,updated=? WHERE attempt_id=?', + (state,resource,'上次执行中断;未自动重跑' if resource=='released' else '上次原生进程退出缺少证明,保留现场',now(),identity)) + self.repo._touch(self.repo.item(row['item_id'])['task_id']) + result['blocked' if resource=='unknown' else 'interrupted']+=1 + self.repo.connection.execute("UPDATE verification_runs SET state='interrupted' WHERE state IN ('queued','running','stopping')") + return result + + def begin_shutdown(self): + self.closing=True + with self.repo.store._lock: + runs=[r[0] for r in self.repo.connection.execute("SELECT run_id FROM verification_runs WHERE state IN ('queued','running','stopping')")] + for identity in runs:self.stop(identity,request_id='shutdown:'+identity) + with self.lock: + for event in self.events.values():event.set() diff --git a/src/dx_productivity/trial_task_center/verification/runner.py b/src/dx_productivity/trial_task_center/verification/runner.py index 886a558..9dd8d3d 100644 --- a/src/dx_productivity/trial_task_center/verification/runner.py +++ b/src/dx_productivity/trial_task_center/verification/runner.py @@ -43,6 +43,8 @@ def run_attempt(service,attempt_id,*,stop_event): with service.repo.store.transaction(): service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",('原生资源退出未知',attempt_id)) return {'published':False,'blocked':True} + from ..batch_resources import write_record + write_record(output/'resource-released.json',{'owner_id':attempt['owner_id'],'resource_state':'released'}) check_cancelled(stop_event) expected=service.artifacts.read(payload['expectation_ref'],'expectations') if payload['expectation_ref'] else None from .rules import evaluate @@ -66,11 +68,17 @@ def run_attempt(service,attempt_id,*,stop_event): 'input_digest':payload['payload_digest'],'raw':raw}) ref=service.artifacts.put('result',result) result['result_ref']=ref + write_record(output/'result-ready.json',{'result_ref':ref}) check_cancelled(stop_event) published=service.repo.finish(attempt_id,generation=attempt['generation'],result=result) return dict(result,published=published) except Exception as error: if attempt: + from ..native_cli import _UNCERTAIN_SCOPES + if _UNCERTAIN_SCOPES: + with service.repo.store.transaction(): + service.repo.connection.execute("UPDATE verification_attempts SET state='stopping',resource_state='unknown',reason=? WHERE attempt_id=?",(str(error),attempt_id)) + return {'published':False,'blocked':True} cancelled=getattr(error,'code','')=='operation_cancelled' result={'attempt_state':'cancelled' if cancelled else 'failed','resource_state':'released', 'verdict':'INCOMPLETE','reason':str(error),'checks':[],'issues':[]} diff --git a/src/dx_productivity/trial_task_center/verification/service.py b/src/dx_productivity/trial_task_center/verification/service.py index eed783f..e56cc5f 100644 --- a/src/dx_productivity/trial_task_center/verification/service.py +++ b/src/dx_productivity/trial_task_center/verification/service.py @@ -19,6 +19,28 @@ class VerificationService: self._intake_lock = RLock() from .queries import VerificationQueries self.queries = VerificationQueries(self.repo) + from .control import VerificationControl + self.control = VerificationControl(self) + self._recovered=False + + def initialize(self): + with self._intake_lock: + if not self._recovered: + self.recover() + self._recovered=True + return self.reconcile_deliveries() + + @property + def busy(self): return self.control.busy + + def start(self, task_id, *, selection=None, request_id): + return self.control.start(task_id,selection=selection,request_id=request_id) + + def stop(self, run_id, *, request_id): return self.control.stop(run_id,request_id=request_id) + def rerun(self, item_id, version, *, request_id): + return self.start(self.repo.item(item_id)['task_id'],selection=((item_id,version),),request_id=request_id) + def recover(self): return self.control.recover() + def begin_shutdown(self): return self.control.begin_shutdown() def list_tasks(self, **kwargs): return self.queries.list_tasks(**kwargs) def read_task(self, task_id, **kwargs): return self.queries.read_task(task_id, **kwargs) diff --git a/tests/ui/test_verification_control.py b/tests/ui/test_verification_control.py new file mode 100644 index 0000000..3089636 --- /dev/null +++ b/tests/ui/test_verification_control.py @@ -0,0 +1,8 @@ +def test_close_and_controls_do_not_block_new_task_creation(ui,qtbot): + page=ui.window.verification_panel + task=ui.service.verification.create_task('任务',request_id='control')['task_id'] + page.open_task(task) + qtbot.waitUntil(lambda:not page.controller.task_query.busy) + assert page.start_button.isEnabled() and page.new_button.isEnabled() + page.begin_shutdown() + assert page.new_button.isEnabled() diff --git a/tests/v1_1/test_verification_control.py b/tests/v1_1/test_verification_control.py new file mode 100644 index 0000000..2f5d1d0 --- /dev/null +++ b/tests/v1_1/test_verification_control.py @@ -0,0 +1,59 @@ +import time +from threading import Event +from test_independent_verification_intake import service +from test_independent_verification_runner import imported +from test_native_cli_netlist import graph + + +def wait_idle(service): + deadline=time.monotonic()+5 + while service.busy and time.monotonic()