feat: receive verification inputs and import project pairs

This commit is contained in:
阿白叔 committed 2026-11-17 15:36:17 +08:00
1 parent f5d39f6261
commit 76074f8d11
5 files changed
+324

No files matched your search

@@ -152,6 +152,15 @@ def _finish_execution_run(service, root, task_ids, stop, run_id, result, *, base
receipt['reporting_errors'].append({'code': getattr(error, 'code', 'schedule_receipt_unavailable')})
_persist_execution_receipt(service, journal, run_id, receipt, baseline)
raise
# Optional downstream delivery follows the durable replacement receipt.
# Delivery failure must never turn a saved replacement into a repeat request.
try:
verification = getattr(service, 'verification', None)
if verification is not None:
verification.receive_successful_results(receipt, submission_id='run:' + run_id)
except Exception as error:
receipt['reporting_errors'].append({'code': 'verification_delivery_pending',
'error': type(error).__name__})
return receipt
@@ -102,6 +102,13 @@ def _default_path_opener(path: Path) -> None:
class TaskCenterService:
"""The only API used by UI workers and widgets."""
@property
def verification(self):
if not hasattr(self, '_verification_service'):
from .verification.service import VerificationService
self._verification_service = VerificationService(self._store, self._logging.root)
return self._verification_service
def __init__(
self,
store: TaskStore,
@@ -0,0 +1,86 @@
"""Managed immutable inputs; native tools never receive arbitrary live paths."""
import json
from pathlib import Path
import shutil
from uuid import uuid4
from .. import native_netlist
from ..batch_resources import write_record
from ..operation_control import check_cancelled
from ..native_inventory import _project_options
from .contracts import artifact_ref, fail
class ArtifactStore:
def __init__(self, data_root):
self.data_root = Path(data_root).resolve()
self.root = self.data_root/'verification'/'artifacts'
def directory(self, identity):
artifact_ref({'manifest_id':identity,'sha256':'0'*64})
path = self.root/identity
if path.resolve().parent != self.root.resolve(): fail('证据目录越界。')
return path
def put(self, kind, payload, *, identity=None):
identity = identity or uuid4().hex
directory = self.directory(identity); directory.mkdir(parents=True,exist_ok=True)
manifest = directory/'manifest.json'
if manifest.exists(): fail('不可覆盖已有校验证据。')
write_record(manifest, {'schema_version':1,'kind':kind,'payload':payload})
return {'manifest_id':identity,'sha256':native_netlist._hash(manifest)}
def read(self, ref, kind):
artifact_ref(ref)
if not ref: fail('缺少校验证据。')
directory = self.directory(ref['manifest_id'])
path = directory/'manifest.json'
try:
if native_netlist._hash(path) != ref['sha256']: fail('证据清单摘要不符。')
manifest = json.loads(path.read_text(encoding='utf-8'))
if manifest.get('schema_version') != 1 or manifest.get('kind') != kind: fail('证据类型不符。')
payload = manifest['payload']
if kind == 'snapshot':
capture_root = Path(payload['root']).resolve()
if capture_root != (directory/'capture').resolve(): fail('输入副本归属不符。')
for relative in payload['capture_hashes']:
if not (capture_root/relative).resolve().is_relative_to(capture_root): fail('输入文件越界。')
for key,relative in (('project_path','input.prj'),('database_path','database/icdb.dat')):
if Path(payload[key]).resolve() != (capture_root/relative).resolve(): fail('输入文件身份不符。')
native_netlist._check_capture(payload)
return payload
except (OSError,ValueError,KeyError,TypeError) as error:
fail(f'校验证据不可用:{error}', 'verification_evidence')
def capture(self, project, *, stop_event):
check_cancelled(stop_event)
identity = uuid4().hex
capture = native_netlist.capture_project(Path(project),self.directory(identity)/'capture')
check_cancelled(stop_event)
return self.put('snapshot',capture,identity=identity)
def adopt(self, capture):
root = Path(capture['root']).resolve()
if not root.is_relative_to(self.data_root): fail('交接副本不属于当前任务库。')
native_netlist._check_capture(capture)
identity = uuid4().hex
destination = self.directory(identity)/'capture'
destination.mkdir(parents=True)
for relative, sha in capture['capture_hashes'].items():
source = (root/relative).resolve(); target=(destination/relative).resolve()
if not source.is_relative_to(root) or not target.is_relative_to(destination.resolve()): fail('交接文件路径越界。')
target.parent.mkdir(parents=True,exist_ok=True); shutil.copyfile(source,target)
if native_netlist._hash(target) != sha: fail('交接副本在复制时变化。')
value = dict(capture,root=str(destination),project_path=str(destination/'input.prj'),
database_path=str(destination/'database/icdb.dat'),manifest_path=str(destination/'capture.json'))
native_netlist._check_capture(capture)
write_record(destination/'capture.json',value)
return self.put('snapshot',value,identity=identity)
def validate_pair(before, after):
before,after = Path(before).resolve(strict=True),Path(after).resolve(strict=True)
if before==after or before.samefile(after): fail('请选择不同的前后工程。','verification_same_source')
before_db,after_db = _project_options(before)[3],_project_options(after)[3]
if Path(before_db).samefile(after_db): fail('两个工程引用同一个数据库。','verification_same_source')
return before,after
@@ -0,0 +1,136 @@
"""One interface for standalone verification and successful-result handoff."""
import json
from pathlib import Path
from threading import RLock
from uuid import uuid4
from ..batch_resources import write_record
from ..operation_control import check_cancelled
from .contracts import default_plan, digest, fail, identifier
from .intake import ArtifactStore, validate_pair
from .store import VerificationRepository
class VerificationService:
def __init__(self, task_store, data_root):
self.repo = VerificationRepository(task_store)
self.data_root = Path(data_root).resolve()
self.artifacts = ArtifactStore(self.data_root)
self._intake_lock = RLock()
def create_task(self, name, *, request_id):
return self.repo.create_task(name,request_id=request_id)
def publish_inputs(self, task_id, inputs, *, submission_id):
body = {'task_id':task_id,'inputs':inputs}
with self.repo.store._lock:
old = self.repo._request('ingest',submission_id,body)
if old is not None: return old
normalized=[]
for value in inputs:
if not isinstance(value,dict):
normalized.append(value)
continue
row=dict(value)
try:
for field,kind in (('before_ref','snapshot'),('after_ref','snapshot'),('expectation_ref','expectations')):
if row.get(field): self.artifacts.read(row[field],kind)
except (ValueError,OSError) as error:
row['unavailable_reason']=str(error)
normalized.append(row)
with self.repo.store.transaction():
old=self.repo._request('ingest',submission_id,body)
if old is not None:return old
result=self.repo.publish(task_id,tuple(normalized),submission_id='ingest:'+submission_id)
return self.repo._remember('ingest',submission_id,body,result)
def import_pair(self, task_id, before, after, *, request_id, stop_event):
body=[task_id,str(before),str(after)]
with self._intake_lock:
with self.repo.store._lock:
old=self.repo._request('import',request_id,body)
if old is not None:return old
check_cancelled(stop_event)
self.repo.task(task_id)
try: before,after=validate_pair(before,after)
except OSError as error: fail(f'工程文件不可用:{error}')
left=self.artifacts.capture(before,stop_event=stop_event)
right=self.artifacts.capture(after,stop_event=stop_event)
row={'item_key':uuid4().hex,'project_key':digest([str(before),str(after)]),
'display_name':after.stem,'input_version':1,'before_ref':left,'after_ref':right,
'expectation_ref':None,'check_plan':default_plan()}
check_cancelled(stop_event)
with self.repo.store.transaction():
result=self.publish_inputs(task_id,(row,),submission_id='import:'+request_id)
return self.repo._remember('import',request_id,body,result)
def receive_successful_results(self, result_ref, *, submission_id):
with self._intake_lock:
with self.repo.store._lock:
old=self.repo._request('handoff',submission_id,result_ref)
if old is not None:return old
reports=result_ref.get('netlist_verification') or {}
candidates=[]
for key,report in reports.items():
execution=report.get('execution') or {}
if execution.get('state')=='completed' and (execution.get('saved_items') or 0)>0:
candidates.append((key,report))
# A successful save without optional netlist evidence is still visible.
for task in result_ref.get('tasks',()):
if task.get('state')=='completed' and (task.get('saved_items') or 0)>0 and task['task_id'] not in reports:
candidates.append((task['task_id'],{'execution':{'state':'completed','saved_items':task['saved_items']},
'project_path':task.get('name',task['task_id'])}))
empty={key:[] for key in ('accepted','duplicates','stale_ignored','rejected')}
if not candidates:return empty
run_key=identifier(str(result_ref.get('queue_id') or result_ref.get('run_id') or submission_id))
task=self.create_task('校验任务',request_id='handoff-task:'+run_key)['task_id']
rows=[]
for key,report in candidates:
delivery_key=digest([run_key,key,report])
with self.repo.store._lock:
prior=self.repo._request('handoff-row',delivery_key,report)
if prior is not None:
rows.append(prior)
continue
refs={}; reason=''
try:
for phase in ('before','after'):
evidence=report.get(phase) or {}
path=Path(evidence.get('export_path','')).resolve()
if not path.is_relative_to(self.data_root) or not path.is_file(): fail('缺少可信前后副本。')
from ..native_netlist import _hash
if _hash(path)!=evidence.get('export_sha256'):fail('交接导出摘要不符。')
exported=json.loads(path.read_text(encoding='utf-8'))
refs[phase]=self.artifacts.adopt(exported['capture'])
except (ValueError,OSError,KeyError,TypeError) as error: reason=str(error)
expected=None
if not reason and 'expected_changes' in report:
expected=self.artifacts.put('expectations',{'changes':report['expected_changes'],
'symbols':report.get('expected_symbols',{}),'execution':report.get('execution',{})})
plan=default_plan()
if expected:plan['required_checks'].append('declared_attributes')
row={'item_key':str(key),'project_key':str(report.get('project_path') or key),
'display_name':Path(str(report.get('project_path') or key)).stem,'input_version':1,
'before_ref':refs.get('before'),'after_ref':refs.get('after'),
'expectation_ref':expected,'check_plan':plan,'unavailable_reason':reason}
with self.repo.store.transaction():
self.repo._remember('handoff-row',delivery_key,report,row)
rows.append(row)
with self.repo.store.transaction():
result=self.publish_inputs(task,tuple(rows),submission_id='handoff:'+submission_id)
result['task_id']=task
return self.repo._remember('handoff',submission_id,result_ref,result)
def reconcile_deliveries(self, *, limit=100):
result={'accepted':0,'errors':[]}
# Completed queue checkpoints are durable and exist independently of a UI callback.
directory=self.data_root/'task-queues'
for path in sorted(directory.glob('*.json'),key=lambda p:p.stat().st_mtime,reverse=True)[:limit]:
try:
value=json.loads(path.read_text(encoding='utf-8'))
value.setdefault('run_id',path.stem)
record=self.receive_successful_results(value,submission_id='checkpoint:'+digest(value))
result['accepted']+=len(record['accepted'])
except (ValueError,OSError,KeyError,TypeError) as error:
result['errors'].append({'record':path.name,'reason':str(error)})
return result
@@ -0,0 +1,86 @@
from pathlib import Path
from threading import Event
import json
import pytest
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.errors import TaskCenterError
from test_native_cli_netlist import project_fixture
@pytest.fixture
def service(tmp_path):
from dx_productivity.trial_task_center.verification.service import VerificationService
store = TaskStore(tmp_path/'tasks.db')
try: yield VerificationService(store,tmp_path/'data')
finally: store.close()
def test_pair_import_creates_same_input_contract_without_replacement(service,tmp_path):
a = tmp_path/'a'; b = tmp_path/'b'; a.mkdir(); b.mkdir()
before,after = project_fixture(a),project_fixture(b)
task = service.create_task('独立任务',request_id='create')['task_id']
result = service.import_pair(task,before,after,request_id='import',stop_event=Event())
item = result['accepted'][0]['item_id']
row = service.repo.revision(item,1)['payload']
assert row['input_state']=='ready' and row['expectation_ref'] is None
assert service.import_pair(task,before,after,request_id='import',stop_event=Event())==result
assert service.repo.latest_attempt(item,1) is None
before_capture = service.artifacts.read(row['before_ref'],'snapshot')
assert Path(before_capture['database_path']).read_bytes()==b'saved-database'
Path(before_capture['database_path']).write_bytes(b'tampered')
with pytest.raises(TaskCenterError): service.artifacts.read(row['before_ref'],'snapshot')
def test_same_source_and_cancel_do_not_publish_items(service,tmp_path):
project = project_fixture(tmp_path)
task = service.create_task('任务',request_id='c')['task_id']
with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='i',stop_event=Event())
stopped=Event(); stopped.set()
with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='j',stop_event=stopped)
assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==0
def test_missing_handoff_evidence_remains_visible_and_duplicate_delivery_does_not_repeat(service):
report={'task_id':'execution-task','project_path':'F:/board.prj',
'execution':{'state':'completed','saved_items':1},'status':'unavailable','reason':'capture failed'}
result=service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff')
item=result['accepted'][0]['item_id']
assert service.repo.revision(item,1)['payload']['input_state']=='unavailable'
assert service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff')==result
assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==1
def test_unsaved_handoff_is_not_accepted_as_success(service):
report={'task_id':'e','project_path':'F:/board.prj','execution':{'state':'failed','saved_items':0}}
result=service.receive_successful_results({'run_id':'r','netlist_verification':{'e':report}},submission_id='bad')
assert not result['accepted']
def test_artifact_reference_cannot_escape_managed_directory(service):
with pytest.raises(TaskCenterError):
service.artifacts.read({'manifest_id':'../../secret','sha256':'1'*64},'snapshot')
def test_malformed_row_does_not_block_good_rows(service):
task=service.create_task('任务',request_id='c')['task_id']
result=service.publish_inputs(task,(None,{}),submission_id='mixed')
assert len(result['rejected'])==2
def test_complete_delivery_can_be_reconciled_under_another_notification(service,tmp_path):
from dx_productivity.trial_task_center.native_netlist import capture_project, _hash
reports={}
for phase in ('before','after'):
folder=tmp_path/phase; folder.mkdir()
capture=capture_project(project_fixture(folder),service.data_root/phase/'capture')
path=service.data_root/phase/'export.json'
path.write_text(json.dumps({'capture':capture}),encoding='utf-8')
reports[phase]={'export_path':str(path),'export_sha256':_hash(path)}
reports.update(project_path='board.prj',execution={'state':'completed','saved_items':1},expected_changes={})
receipt={'run_id':'r','queue_id':'q','netlist_verification':{'e':reports}}
first=service.receive_successful_results(receipt,submission_id='receipt')
checkpoint=dict(receipt); checkpoint['run_id']='q'
second=service.receive_successful_results(checkpoint,submission_id='checkpoint')
assert len(first['accepted'])==1
assert len(second['duplicates'])==1 and not second['rejected']
assert second['task_id']==first['task_id']