From 76074f8d110f183d16a116da85dcb81f4ec28061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Tue, 17 Nov 2026 15:36:17 +0800 Subject: [PATCH] feat: receive verification inputs and import project pairs --- src/dx_productivity/dsh/execution_runs.py | 9 ++ .../trial_task_center/service.py | 7 + .../trial_task_center/verification/intake.py | 86 +++++++++++ .../trial_task_center/verification/service.py | 136 ++++++++++++++++++ .../test_independent_verification_intake.py | 86 +++++++++++ 5 files changed, 324 insertions(+) create mode 100644 src/dx_productivity/trial_task_center/verification/intake.py create mode 100644 src/dx_productivity/trial_task_center/verification/service.py create mode 100644 tests/v1_1/test_independent_verification_intake.py diff --git a/src/dx_productivity/dsh/execution_runs.py b/src/dx_productivity/dsh/execution_runs.py index 978faf8..9f05a39 100644 --- a/src/dx_productivity/dsh/execution_runs.py +++ b/src/dx_productivity/dsh/execution_runs.py @@ -152,6 +152,15 @@ def _finish_execution_run(service, root, task_ids, stop, run_id, result, *, base receipt['reporting_errors'].append({'code': getattr(error, 'code', 'schedule_receipt_unavailable')}) _persist_execution_receipt(service, journal, run_id, receipt, baseline) raise + # Optional downstream delivery follows the durable replacement receipt. + # Delivery failure must never turn a saved replacement into a repeat request. + try: + verification = getattr(service, 'verification', None) + if verification is not None: + verification.receive_successful_results(receipt, submission_id='run:' + run_id) + except Exception as error: + receipt['reporting_errors'].append({'code': 'verification_delivery_pending', + 'error': type(error).__name__}) return receipt diff --git a/src/dx_productivity/trial_task_center/service.py b/src/dx_productivity/trial_task_center/service.py index 770224c..e295469 100644 --- a/src/dx_productivity/trial_task_center/service.py +++ b/src/dx_productivity/trial_task_center/service.py @@ -102,6 +102,13 @@ def _default_path_opener(path: Path) -> None: class TaskCenterService: """The only API used by UI workers and widgets.""" + @property + def verification(self): + if not hasattr(self, '_verification_service'): + from .verification.service import VerificationService + self._verification_service = VerificationService(self._store, self._logging.root) + return self._verification_service + def __init__( self, store: TaskStore, diff --git a/src/dx_productivity/trial_task_center/verification/intake.py b/src/dx_productivity/trial_task_center/verification/intake.py new file mode 100644 index 0000000..cc4922e --- /dev/null +++ b/src/dx_productivity/trial_task_center/verification/intake.py @@ -0,0 +1,86 @@ +"""Managed immutable inputs; native tools never receive arbitrary live paths.""" +import json +from pathlib import Path +import shutil +from uuid import uuid4 + +from .. import native_netlist +from ..batch_resources import write_record +from ..operation_control import check_cancelled +from ..native_inventory import _project_options +from .contracts import artifact_ref, fail + + +class ArtifactStore: + def __init__(self, data_root): + self.data_root = Path(data_root).resolve() + self.root = self.data_root/'verification'/'artifacts' + + def directory(self, identity): + artifact_ref({'manifest_id':identity,'sha256':'0'*64}) + path = self.root/identity + if path.resolve().parent != self.root.resolve(): fail('证据目录越界。') + return path + + def put(self, kind, payload, *, identity=None): + identity = identity or uuid4().hex + directory = self.directory(identity); directory.mkdir(parents=True,exist_ok=True) + manifest = directory/'manifest.json' + if manifest.exists(): fail('不可覆盖已有校验证据。') + write_record(manifest, {'schema_version':1,'kind':kind,'payload':payload}) + return {'manifest_id':identity,'sha256':native_netlist._hash(manifest)} + + def read(self, ref, kind): + artifact_ref(ref) + if not ref: fail('缺少校验证据。') + directory = self.directory(ref['manifest_id']) + path = directory/'manifest.json' + try: + if native_netlist._hash(path) != ref['sha256']: fail('证据清单摘要不符。') + manifest = json.loads(path.read_text(encoding='utf-8')) + if manifest.get('schema_version') != 1 or manifest.get('kind') != kind: fail('证据类型不符。') + payload = manifest['payload'] + if kind == 'snapshot': + capture_root = Path(payload['root']).resolve() + if capture_root != (directory/'capture').resolve(): fail('输入副本归属不符。') + for relative in payload['capture_hashes']: + if not (capture_root/relative).resolve().is_relative_to(capture_root): fail('输入文件越界。') + for key,relative in (('project_path','input.prj'),('database_path','database/icdb.dat')): + if Path(payload[key]).resolve() != (capture_root/relative).resolve(): fail('输入文件身份不符。') + native_netlist._check_capture(payload) + return payload + except (OSError,ValueError,KeyError,TypeError) as error: + fail(f'校验证据不可用:{error}', 'verification_evidence') + + def capture(self, project, *, stop_event): + check_cancelled(stop_event) + identity = uuid4().hex + capture = native_netlist.capture_project(Path(project),self.directory(identity)/'capture') + check_cancelled(stop_event) + return self.put('snapshot',capture,identity=identity) + + def adopt(self, capture): + root = Path(capture['root']).resolve() + if not root.is_relative_to(self.data_root): fail('交接副本不属于当前任务库。') + native_netlist._check_capture(capture) + identity = uuid4().hex + destination = self.directory(identity)/'capture' + destination.mkdir(parents=True) + for relative, sha in capture['capture_hashes'].items(): + source = (root/relative).resolve(); target=(destination/relative).resolve() + if not source.is_relative_to(root) or not target.is_relative_to(destination.resolve()): fail('交接文件路径越界。') + target.parent.mkdir(parents=True,exist_ok=True); shutil.copyfile(source,target) + if native_netlist._hash(target) != sha: fail('交接副本在复制时变化。') + value = dict(capture,root=str(destination),project_path=str(destination/'input.prj'), + database_path=str(destination/'database/icdb.dat'),manifest_path=str(destination/'capture.json')) + native_netlist._check_capture(capture) + write_record(destination/'capture.json',value) + return self.put('snapshot',value,identity=identity) + + +def validate_pair(before, after): + before,after = Path(before).resolve(strict=True),Path(after).resolve(strict=True) + if before==after or before.samefile(after): fail('请选择不同的前后工程。','verification_same_source') + before_db,after_db = _project_options(before)[3],_project_options(after)[3] + if Path(before_db).samefile(after_db): fail('两个工程引用同一个数据库。','verification_same_source') + return before,after diff --git a/src/dx_productivity/trial_task_center/verification/service.py b/src/dx_productivity/trial_task_center/verification/service.py new file mode 100644 index 0000000..afd407b --- /dev/null +++ b/src/dx_productivity/trial_task_center/verification/service.py @@ -0,0 +1,136 @@ +"""One interface for standalone verification and successful-result handoff.""" +import json +from pathlib import Path +from threading import RLock +from uuid import uuid4 + +from ..batch_resources import write_record +from ..operation_control import check_cancelled +from .contracts import default_plan, digest, fail, identifier +from .intake import ArtifactStore, validate_pair +from .store import VerificationRepository + + +class VerificationService: + def __init__(self, task_store, data_root): + self.repo = VerificationRepository(task_store) + self.data_root = Path(data_root).resolve() + self.artifacts = ArtifactStore(self.data_root) + self._intake_lock = RLock() + + def create_task(self, name, *, request_id): + return self.repo.create_task(name,request_id=request_id) + + def publish_inputs(self, task_id, inputs, *, submission_id): + body = {'task_id':task_id,'inputs':inputs} + with self.repo.store._lock: + old = self.repo._request('ingest',submission_id,body) + if old is not None: return old + normalized=[] + for value in inputs: + if not isinstance(value,dict): + normalized.append(value) + continue + row=dict(value) + try: + for field,kind in (('before_ref','snapshot'),('after_ref','snapshot'),('expectation_ref','expectations')): + if row.get(field): self.artifacts.read(row[field],kind) + except (ValueError,OSError) as error: + row['unavailable_reason']=str(error) + normalized.append(row) + with self.repo.store.transaction(): + old=self.repo._request('ingest',submission_id,body) + if old is not None:return old + result=self.repo.publish(task_id,tuple(normalized),submission_id='ingest:'+submission_id) + return self.repo._remember('ingest',submission_id,body,result) + + def import_pair(self, task_id, before, after, *, request_id, stop_event): + body=[task_id,str(before),str(after)] + with self._intake_lock: + with self.repo.store._lock: + old=self.repo._request('import',request_id,body) + if old is not None:return old + check_cancelled(stop_event) + self.repo.task(task_id) + try: before,after=validate_pair(before,after) + except OSError as error: fail(f'工程文件不可用:{error}') + left=self.artifacts.capture(before,stop_event=stop_event) + right=self.artifacts.capture(after,stop_event=stop_event) + row={'item_key':uuid4().hex,'project_key':digest([str(before),str(after)]), + 'display_name':after.stem,'input_version':1,'before_ref':left,'after_ref':right, + 'expectation_ref':None,'check_plan':default_plan()} + check_cancelled(stop_event) + with self.repo.store.transaction(): + result=self.publish_inputs(task_id,(row,),submission_id='import:'+request_id) + return self.repo._remember('import',request_id,body,result) + + def receive_successful_results(self, result_ref, *, submission_id): + with self._intake_lock: + with self.repo.store._lock: + old=self.repo._request('handoff',submission_id,result_ref) + if old is not None:return old + reports=result_ref.get('netlist_verification') or {} + candidates=[] + for key,report in reports.items(): + execution=report.get('execution') or {} + if execution.get('state')=='completed' and (execution.get('saved_items') or 0)>0: + candidates.append((key,report)) + # A successful save without optional netlist evidence is still visible. + for task in result_ref.get('tasks',()): + if task.get('state')=='completed' and (task.get('saved_items') or 0)>0 and task['task_id'] not in reports: + candidates.append((task['task_id'],{'execution':{'state':'completed','saved_items':task['saved_items']}, + 'project_path':task.get('name',task['task_id'])})) + empty={key:[] for key in ('accepted','duplicates','stale_ignored','rejected')} + if not candidates:return empty + run_key=identifier(str(result_ref.get('queue_id') or result_ref.get('run_id') or submission_id)) + task=self.create_task('校验任务',request_id='handoff-task:'+run_key)['task_id'] + rows=[] + for key,report in candidates: + delivery_key=digest([run_key,key,report]) + with self.repo.store._lock: + prior=self.repo._request('handoff-row',delivery_key,report) + if prior is not None: + rows.append(prior) + continue + refs={}; reason='' + try: + for phase in ('before','after'): + evidence=report.get(phase) or {} + path=Path(evidence.get('export_path','')).resolve() + if not path.is_relative_to(self.data_root) or not path.is_file(): fail('缺少可信前后副本。') + from ..native_netlist import _hash + if _hash(path)!=evidence.get('export_sha256'):fail('交接导出摘要不符。') + exported=json.loads(path.read_text(encoding='utf-8')) + refs[phase]=self.artifacts.adopt(exported['capture']) + except (ValueError,OSError,KeyError,TypeError) as error: reason=str(error) + expected=None + if not reason and 'expected_changes' in report: + expected=self.artifacts.put('expectations',{'changes':report['expected_changes'], + 'symbols':report.get('expected_symbols',{}),'execution':report.get('execution',{})}) + plan=default_plan() + if expected:plan['required_checks'].append('declared_attributes') + row={'item_key':str(key),'project_key':str(report.get('project_path') or key), + 'display_name':Path(str(report.get('project_path') or key)).stem,'input_version':1, + 'before_ref':refs.get('before'),'after_ref':refs.get('after'), + 'expectation_ref':expected,'check_plan':plan,'unavailable_reason':reason} + with self.repo.store.transaction(): + self.repo._remember('handoff-row',delivery_key,report,row) + rows.append(row) + with self.repo.store.transaction(): + result=self.publish_inputs(task,tuple(rows),submission_id='handoff:'+submission_id) + result['task_id']=task + return self.repo._remember('handoff',submission_id,result_ref,result) + + def reconcile_deliveries(self, *, limit=100): + result={'accepted':0,'errors':[]} + # Completed queue checkpoints are durable and exist independently of a UI callback. + directory=self.data_root/'task-queues' + for path in sorted(directory.glob('*.json'),key=lambda p:p.stat().st_mtime,reverse=True)[:limit]: + try: + value=json.loads(path.read_text(encoding='utf-8')) + value.setdefault('run_id',path.stem) + record=self.receive_successful_results(value,submission_id='checkpoint:'+digest(value)) + result['accepted']+=len(record['accepted']) + except (ValueError,OSError,KeyError,TypeError) as error: + result['errors'].append({'record':path.name,'reason':str(error)}) + return result diff --git a/tests/v1_1/test_independent_verification_intake.py b/tests/v1_1/test_independent_verification_intake.py new file mode 100644 index 0000000..a46300f --- /dev/null +++ b/tests/v1_1/test_independent_verification_intake.py @@ -0,0 +1,86 @@ +from pathlib import Path +from threading import Event +import json +import pytest +from dx_productivity.trial_task_center.store import TaskStore +from dx_productivity.trial_task_center.errors import TaskCenterError +from test_native_cli_netlist import project_fixture + + +@pytest.fixture +def service(tmp_path): + from dx_productivity.trial_task_center.verification.service import VerificationService + store = TaskStore(tmp_path/'tasks.db') + try: yield VerificationService(store,tmp_path/'data') + finally: store.close() + + +def test_pair_import_creates_same_input_contract_without_replacement(service,tmp_path): + a = tmp_path/'a'; b = tmp_path/'b'; a.mkdir(); b.mkdir() + before,after = project_fixture(a),project_fixture(b) + task = service.create_task('独立任务',request_id='create')['task_id'] + result = service.import_pair(task,before,after,request_id='import',stop_event=Event()) + item = result['accepted'][0]['item_id'] + row = service.repo.revision(item,1)['payload'] + assert row['input_state']=='ready' and row['expectation_ref'] is None + assert service.import_pair(task,before,after,request_id='import',stop_event=Event())==result + assert service.repo.latest_attempt(item,1) is None + before_capture = service.artifacts.read(row['before_ref'],'snapshot') + assert Path(before_capture['database_path']).read_bytes()==b'saved-database' + Path(before_capture['database_path']).write_bytes(b'tampered') + with pytest.raises(TaskCenterError): service.artifacts.read(row['before_ref'],'snapshot') + + +def test_same_source_and_cancel_do_not_publish_items(service,tmp_path): + project = project_fixture(tmp_path) + task = service.create_task('任务',request_id='c')['task_id'] + with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='i',stop_event=Event()) + stopped=Event(); stopped.set() + with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='j',stop_event=stopped) + assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==0 + + +def test_missing_handoff_evidence_remains_visible_and_duplicate_delivery_does_not_repeat(service): + report={'task_id':'execution-task','project_path':'F:/board.prj', + 'execution':{'state':'completed','saved_items':1},'status':'unavailable','reason':'capture failed'} + result=service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff') + item=result['accepted'][0]['item_id'] + assert service.repo.revision(item,1)['payload']['input_state']=='unavailable' + assert service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff')==result + assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==1 + + +def test_unsaved_handoff_is_not_accepted_as_success(service): + report={'task_id':'e','project_path':'F:/board.prj','execution':{'state':'failed','saved_items':0}} + result=service.receive_successful_results({'run_id':'r','netlist_verification':{'e':report}},submission_id='bad') + assert not result['accepted'] + + +def test_artifact_reference_cannot_escape_managed_directory(service): + with pytest.raises(TaskCenterError): + service.artifacts.read({'manifest_id':'../../secret','sha256':'1'*64},'snapshot') + + +def test_malformed_row_does_not_block_good_rows(service): + task=service.create_task('任务',request_id='c')['task_id'] + result=service.publish_inputs(task,(None,{}),submission_id='mixed') + assert len(result['rejected'])==2 + + +def test_complete_delivery_can_be_reconciled_under_another_notification(service,tmp_path): + from dx_productivity.trial_task_center.native_netlist import capture_project, _hash + reports={} + for phase in ('before','after'): + folder=tmp_path/phase; folder.mkdir() + capture=capture_project(project_fixture(folder),service.data_root/phase/'capture') + path=service.data_root/phase/'export.json' + path.write_text(json.dumps({'capture':capture}),encoding='utf-8') + reports[phase]={'export_path':str(path),'export_sha256':_hash(path)} + reports.update(project_path='board.prj',execution={'state':'completed','saved_items':1},expected_changes={}) + receipt={'run_id':'r','queue_id':'q','netlist_verification':{'e':reports}} + first=service.receive_successful_results(receipt,submission_id='receipt') + checkpoint=dict(receipt); checkpoint['run_id']='q' + second=service.receive_successful_results(checkpoint,submission_id='checkpoint') + assert len(first['accepted'])==1 + assert len(second['duplicates'])==1 and not second['rejected'] + assert second['task_id']==first['task_id']