feat: receive verification inputs and import project pairs
This commit is contained in:
1 parent
f5d39f6261
commit
76074f8d11
5 files changed
+324
No files matched your search
@@ -152,6 +152,15 @@ def _finish_execution_run(service, root, task_ids, stop, run_id, result, *, base
|
||||
receipt['reporting_errors'].append({'code': getattr(error, 'code', 'schedule_receipt_unavailable')})
|
||||
_persist_execution_receipt(service, journal, run_id, receipt, baseline)
|
||||
raise
|
||||
# Optional downstream delivery follows the durable replacement receipt.
|
||||
# Delivery failure must never turn a saved replacement into a repeat request.
|
||||
try:
|
||||
verification = getattr(service, 'verification', None)
|
||||
if verification is not None:
|
||||
verification.receive_successful_results(receipt, submission_id='run:' + run_id)
|
||||
except Exception as error:
|
||||
receipt['reporting_errors'].append({'code': 'verification_delivery_pending',
|
||||
'error': type(error).__name__})
|
||||
return receipt
|
||||
|
||||
|
||||
|
||||
@@ -102,6 +102,13 @@ def _default_path_opener(path: Path) -> None:
|
||||
class TaskCenterService:
|
||||
"""The only API used by UI workers and widgets."""
|
||||
|
||||
@property
|
||||
def verification(self):
|
||||
if not hasattr(self, '_verification_service'):
|
||||
from .verification.service import VerificationService
|
||||
self._verification_service = VerificationService(self._store, self._logging.root)
|
||||
return self._verification_service
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: TaskStore,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Managed immutable inputs; native tools never receive arbitrary live paths."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from uuid import uuid4
|
||||
|
||||
from .. import native_netlist
|
||||
from ..batch_resources import write_record
|
||||
from ..operation_control import check_cancelled
|
||||
from ..native_inventory import _project_options
|
||||
from .contracts import artifact_ref, fail
|
||||
|
||||
|
||||
class ArtifactStore:
|
||||
def __init__(self, data_root):
|
||||
self.data_root = Path(data_root).resolve()
|
||||
self.root = self.data_root/'verification'/'artifacts'
|
||||
|
||||
def directory(self, identity):
|
||||
artifact_ref({'manifest_id':identity,'sha256':'0'*64})
|
||||
path = self.root/identity
|
||||
if path.resolve().parent != self.root.resolve(): fail('证据目录越界。')
|
||||
return path
|
||||
|
||||
def put(self, kind, payload, *, identity=None):
|
||||
identity = identity or uuid4().hex
|
||||
directory = self.directory(identity); directory.mkdir(parents=True,exist_ok=True)
|
||||
manifest = directory/'manifest.json'
|
||||
if manifest.exists(): fail('不可覆盖已有校验证据。')
|
||||
write_record(manifest, {'schema_version':1,'kind':kind,'payload':payload})
|
||||
return {'manifest_id':identity,'sha256':native_netlist._hash(manifest)}
|
||||
|
||||
def read(self, ref, kind):
|
||||
artifact_ref(ref)
|
||||
if not ref: fail('缺少校验证据。')
|
||||
directory = self.directory(ref['manifest_id'])
|
||||
path = directory/'manifest.json'
|
||||
try:
|
||||
if native_netlist._hash(path) != ref['sha256']: fail('证据清单摘要不符。')
|
||||
manifest = json.loads(path.read_text(encoding='utf-8'))
|
||||
if manifest.get('schema_version') != 1 or manifest.get('kind') != kind: fail('证据类型不符。')
|
||||
payload = manifest['payload']
|
||||
if kind == 'snapshot':
|
||||
capture_root = Path(payload['root']).resolve()
|
||||
if capture_root != (directory/'capture').resolve(): fail('输入副本归属不符。')
|
||||
for relative in payload['capture_hashes']:
|
||||
if not (capture_root/relative).resolve().is_relative_to(capture_root): fail('输入文件越界。')
|
||||
for key,relative in (('project_path','input.prj'),('database_path','database/icdb.dat')):
|
||||
if Path(payload[key]).resolve() != (capture_root/relative).resolve(): fail('输入文件身份不符。')
|
||||
native_netlist._check_capture(payload)
|
||||
return payload
|
||||
except (OSError,ValueError,KeyError,TypeError) as error:
|
||||
fail(f'校验证据不可用:{error}', 'verification_evidence')
|
||||
|
||||
def capture(self, project, *, stop_event):
|
||||
check_cancelled(stop_event)
|
||||
identity = uuid4().hex
|
||||
capture = native_netlist.capture_project(Path(project),self.directory(identity)/'capture')
|
||||
check_cancelled(stop_event)
|
||||
return self.put('snapshot',capture,identity=identity)
|
||||
|
||||
def adopt(self, capture):
|
||||
root = Path(capture['root']).resolve()
|
||||
if not root.is_relative_to(self.data_root): fail('交接副本不属于当前任务库。')
|
||||
native_netlist._check_capture(capture)
|
||||
identity = uuid4().hex
|
||||
destination = self.directory(identity)/'capture'
|
||||
destination.mkdir(parents=True)
|
||||
for relative, sha in capture['capture_hashes'].items():
|
||||
source = (root/relative).resolve(); target=(destination/relative).resolve()
|
||||
if not source.is_relative_to(root) or not target.is_relative_to(destination.resolve()): fail('交接文件路径越界。')
|
||||
target.parent.mkdir(parents=True,exist_ok=True); shutil.copyfile(source,target)
|
||||
if native_netlist._hash(target) != sha: fail('交接副本在复制时变化。')
|
||||
value = dict(capture,root=str(destination),project_path=str(destination/'input.prj'),
|
||||
database_path=str(destination/'database/icdb.dat'),manifest_path=str(destination/'capture.json'))
|
||||
native_netlist._check_capture(capture)
|
||||
write_record(destination/'capture.json',value)
|
||||
return self.put('snapshot',value,identity=identity)
|
||||
|
||||
|
||||
def validate_pair(before, after):
|
||||
before,after = Path(before).resolve(strict=True),Path(after).resolve(strict=True)
|
||||
if before==after or before.samefile(after): fail('请选择不同的前后工程。','verification_same_source')
|
||||
before_db,after_db = _project_options(before)[3],_project_options(after)[3]
|
||||
if Path(before_db).samefile(after_db): fail('两个工程引用同一个数据库。','verification_same_source')
|
||||
return before,after
|
||||
@@ -0,0 +1,136 @@
|
||||
"""One interface for standalone verification and successful-result handoff."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from uuid import uuid4
|
||||
|
||||
from ..batch_resources import write_record
|
||||
from ..operation_control import check_cancelled
|
||||
from .contracts import default_plan, digest, fail, identifier
|
||||
from .intake import ArtifactStore, validate_pair
|
||||
from .store import VerificationRepository
|
||||
|
||||
|
||||
class VerificationService:
|
||||
def __init__(self, task_store, data_root):
|
||||
self.repo = VerificationRepository(task_store)
|
||||
self.data_root = Path(data_root).resolve()
|
||||
self.artifacts = ArtifactStore(self.data_root)
|
||||
self._intake_lock = RLock()
|
||||
|
||||
def create_task(self, name, *, request_id):
|
||||
return self.repo.create_task(name,request_id=request_id)
|
||||
|
||||
def publish_inputs(self, task_id, inputs, *, submission_id):
|
||||
body = {'task_id':task_id,'inputs':inputs}
|
||||
with self.repo.store._lock:
|
||||
old = self.repo._request('ingest',submission_id,body)
|
||||
if old is not None: return old
|
||||
normalized=[]
|
||||
for value in inputs:
|
||||
if not isinstance(value,dict):
|
||||
normalized.append(value)
|
||||
continue
|
||||
row=dict(value)
|
||||
try:
|
||||
for field,kind in (('before_ref','snapshot'),('after_ref','snapshot'),('expectation_ref','expectations')):
|
||||
if row.get(field): self.artifacts.read(row[field],kind)
|
||||
except (ValueError,OSError) as error:
|
||||
row['unavailable_reason']=str(error)
|
||||
normalized.append(row)
|
||||
with self.repo.store.transaction():
|
||||
old=self.repo._request('ingest',submission_id,body)
|
||||
if old is not None:return old
|
||||
result=self.repo.publish(task_id,tuple(normalized),submission_id='ingest:'+submission_id)
|
||||
return self.repo._remember('ingest',submission_id,body,result)
|
||||
|
||||
def import_pair(self, task_id, before, after, *, request_id, stop_event):
|
||||
body=[task_id,str(before),str(after)]
|
||||
with self._intake_lock:
|
||||
with self.repo.store._lock:
|
||||
old=self.repo._request('import',request_id,body)
|
||||
if old is not None:return old
|
||||
check_cancelled(stop_event)
|
||||
self.repo.task(task_id)
|
||||
try: before,after=validate_pair(before,after)
|
||||
except OSError as error: fail(f'工程文件不可用:{error}')
|
||||
left=self.artifacts.capture(before,stop_event=stop_event)
|
||||
right=self.artifacts.capture(after,stop_event=stop_event)
|
||||
row={'item_key':uuid4().hex,'project_key':digest([str(before),str(after)]),
|
||||
'display_name':after.stem,'input_version':1,'before_ref':left,'after_ref':right,
|
||||
'expectation_ref':None,'check_plan':default_plan()}
|
||||
check_cancelled(stop_event)
|
||||
with self.repo.store.transaction():
|
||||
result=self.publish_inputs(task_id,(row,),submission_id='import:'+request_id)
|
||||
return self.repo._remember('import',request_id,body,result)
|
||||
|
||||
def receive_successful_results(self, result_ref, *, submission_id):
|
||||
with self._intake_lock:
|
||||
with self.repo.store._lock:
|
||||
old=self.repo._request('handoff',submission_id,result_ref)
|
||||
if old is not None:return old
|
||||
reports=result_ref.get('netlist_verification') or {}
|
||||
candidates=[]
|
||||
for key,report in reports.items():
|
||||
execution=report.get('execution') or {}
|
||||
if execution.get('state')=='completed' and (execution.get('saved_items') or 0)>0:
|
||||
candidates.append((key,report))
|
||||
# A successful save without optional netlist evidence is still visible.
|
||||
for task in result_ref.get('tasks',()):
|
||||
if task.get('state')=='completed' and (task.get('saved_items') or 0)>0 and task['task_id'] not in reports:
|
||||
candidates.append((task['task_id'],{'execution':{'state':'completed','saved_items':task['saved_items']},
|
||||
'project_path':task.get('name',task['task_id'])}))
|
||||
empty={key:[] for key in ('accepted','duplicates','stale_ignored','rejected')}
|
||||
if not candidates:return empty
|
||||
run_key=identifier(str(result_ref.get('queue_id') or result_ref.get('run_id') or submission_id))
|
||||
task=self.create_task('校验任务',request_id='handoff-task:'+run_key)['task_id']
|
||||
rows=[]
|
||||
for key,report in candidates:
|
||||
delivery_key=digest([run_key,key,report])
|
||||
with self.repo.store._lock:
|
||||
prior=self.repo._request('handoff-row',delivery_key,report)
|
||||
if prior is not None:
|
||||
rows.append(prior)
|
||||
continue
|
||||
refs={}; reason=''
|
||||
try:
|
||||
for phase in ('before','after'):
|
||||
evidence=report.get(phase) or {}
|
||||
path=Path(evidence.get('export_path','')).resolve()
|
||||
if not path.is_relative_to(self.data_root) or not path.is_file(): fail('缺少可信前后副本。')
|
||||
from ..native_netlist import _hash
|
||||
if _hash(path)!=evidence.get('export_sha256'):fail('交接导出摘要不符。')
|
||||
exported=json.loads(path.read_text(encoding='utf-8'))
|
||||
refs[phase]=self.artifacts.adopt(exported['capture'])
|
||||
except (ValueError,OSError,KeyError,TypeError) as error: reason=str(error)
|
||||
expected=None
|
||||
if not reason and 'expected_changes' in report:
|
||||
expected=self.artifacts.put('expectations',{'changes':report['expected_changes'],
|
||||
'symbols':report.get('expected_symbols',{}),'execution':report.get('execution',{})})
|
||||
plan=default_plan()
|
||||
if expected:plan['required_checks'].append('declared_attributes')
|
||||
row={'item_key':str(key),'project_key':str(report.get('project_path') or key),
|
||||
'display_name':Path(str(report.get('project_path') or key)).stem,'input_version':1,
|
||||
'before_ref':refs.get('before'),'after_ref':refs.get('after'),
|
||||
'expectation_ref':expected,'check_plan':plan,'unavailable_reason':reason}
|
||||
with self.repo.store.transaction():
|
||||
self.repo._remember('handoff-row',delivery_key,report,row)
|
||||
rows.append(row)
|
||||
with self.repo.store.transaction():
|
||||
result=self.publish_inputs(task,tuple(rows),submission_id='handoff:'+submission_id)
|
||||
result['task_id']=task
|
||||
return self.repo._remember('handoff',submission_id,result_ref,result)
|
||||
|
||||
def reconcile_deliveries(self, *, limit=100):
|
||||
result={'accepted':0,'errors':[]}
|
||||
# Completed queue checkpoints are durable and exist independently of a UI callback.
|
||||
directory=self.data_root/'task-queues'
|
||||
for path in sorted(directory.glob('*.json'),key=lambda p:p.stat().st_mtime,reverse=True)[:limit]:
|
||||
try:
|
||||
value=json.loads(path.read_text(encoding='utf-8'))
|
||||
value.setdefault('run_id',path.stem)
|
||||
record=self.receive_successful_results(value,submission_id='checkpoint:'+digest(value))
|
||||
result['accepted']+=len(record['accepted'])
|
||||
except (ValueError,OSError,KeyError,TypeError) as error:
|
||||
result['errors'].append({'record':path.name,'reason':str(error)})
|
||||
return result
|
||||
@@ -0,0 +1,86 @@
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
import json
|
||||
import pytest
|
||||
from dx_productivity.trial_task_center.store import TaskStore
|
||||
from dx_productivity.trial_task_center.errors import TaskCenterError
|
||||
from test_native_cli_netlist import project_fixture
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service(tmp_path):
|
||||
from dx_productivity.trial_task_center.verification.service import VerificationService
|
||||
store = TaskStore(tmp_path/'tasks.db')
|
||||
try: yield VerificationService(store,tmp_path/'data')
|
||||
finally: store.close()
|
||||
|
||||
|
||||
def test_pair_import_creates_same_input_contract_without_replacement(service,tmp_path):
|
||||
a = tmp_path/'a'; b = tmp_path/'b'; a.mkdir(); b.mkdir()
|
||||
before,after = project_fixture(a),project_fixture(b)
|
||||
task = service.create_task('独立任务',request_id='create')['task_id']
|
||||
result = service.import_pair(task,before,after,request_id='import',stop_event=Event())
|
||||
item = result['accepted'][0]['item_id']
|
||||
row = service.repo.revision(item,1)['payload']
|
||||
assert row['input_state']=='ready' and row['expectation_ref'] is None
|
||||
assert service.import_pair(task,before,after,request_id='import',stop_event=Event())==result
|
||||
assert service.repo.latest_attempt(item,1) is None
|
||||
before_capture = service.artifacts.read(row['before_ref'],'snapshot')
|
||||
assert Path(before_capture['database_path']).read_bytes()==b'saved-database'
|
||||
Path(before_capture['database_path']).write_bytes(b'tampered')
|
||||
with pytest.raises(TaskCenterError): service.artifacts.read(row['before_ref'],'snapshot')
|
||||
|
||||
|
||||
def test_same_source_and_cancel_do_not_publish_items(service,tmp_path):
|
||||
project = project_fixture(tmp_path)
|
||||
task = service.create_task('任务',request_id='c')['task_id']
|
||||
with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='i',stop_event=Event())
|
||||
stopped=Event(); stopped.set()
|
||||
with pytest.raises(TaskCenterError): service.import_pair(task,project,project,request_id='j',stop_event=stopped)
|
||||
assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==0
|
||||
|
||||
|
||||
def test_missing_handoff_evidence_remains_visible_and_duplicate_delivery_does_not_repeat(service):
|
||||
report={'task_id':'execution-task','project_path':'F:/board.prj',
|
||||
'execution':{'state':'completed','saved_items':1},'status':'unavailable','reason':'capture failed'}
|
||||
result=service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff')
|
||||
item=result['accepted'][0]['item_id']
|
||||
assert service.repo.revision(item,1)['payload']['input_state']=='unavailable'
|
||||
assert service.receive_successful_results({'run_id':'run','netlist_verification':{'execution-task':report}},submission_id='handoff')==result
|
||||
assert service.repo.connection.execute('SELECT COUNT(*) FROM verification_items').fetchone()[0]==1
|
||||
|
||||
|
||||
def test_unsaved_handoff_is_not_accepted_as_success(service):
|
||||
report={'task_id':'e','project_path':'F:/board.prj','execution':{'state':'failed','saved_items':0}}
|
||||
result=service.receive_successful_results({'run_id':'r','netlist_verification':{'e':report}},submission_id='bad')
|
||||
assert not result['accepted']
|
||||
|
||||
|
||||
def test_artifact_reference_cannot_escape_managed_directory(service):
|
||||
with pytest.raises(TaskCenterError):
|
||||
service.artifacts.read({'manifest_id':'../../secret','sha256':'1'*64},'snapshot')
|
||||
|
||||
|
||||
def test_malformed_row_does_not_block_good_rows(service):
|
||||
task=service.create_task('任务',request_id='c')['task_id']
|
||||
result=service.publish_inputs(task,(None,{}),submission_id='mixed')
|
||||
assert len(result['rejected'])==2
|
||||
|
||||
|
||||
def test_complete_delivery_can_be_reconciled_under_another_notification(service,tmp_path):
|
||||
from dx_productivity.trial_task_center.native_netlist import capture_project, _hash
|
||||
reports={}
|
||||
for phase in ('before','after'):
|
||||
folder=tmp_path/phase; folder.mkdir()
|
||||
capture=capture_project(project_fixture(folder),service.data_root/phase/'capture')
|
||||
path=service.data_root/phase/'export.json'
|
||||
path.write_text(json.dumps({'capture':capture}),encoding='utf-8')
|
||||
reports[phase]={'export_path':str(path),'export_sha256':_hash(path)}
|
||||
reports.update(project_path='board.prj',execution={'state':'completed','saved_items':1},expected_changes={})
|
||||
receipt={'run_id':'r','queue_id':'q','netlist_verification':{'e':reports}}
|
||||
first=service.receive_successful_results(receipt,submission_id='receipt')
|
||||
checkpoint=dict(receipt); checkpoint['run_id']='q'
|
||||
second=service.receive_successful_results(checkpoint,submission_id='checkpoint')
|
||||
assert len(first['accepted'])==1
|
||||
assert len(second['duplicates'])==1 and not second['rejected']
|
||||
assert second['task_id']==first['task_id']
|
||||
Reference in new issue
Block a user