test: enforce reproducible verification closed-loop standards

This commit is contained in:
阿白叔 committed 2026-11-18 10:23:14 +08:00
1 parent a74b810786
commit e65bed0408
7 files changed
+556 -4

No files matched your search

@@ -138,6 +138,21 @@ input_record = {
## 8. 交接状态
### N12 后续闭环标准(2026-11-18)
所有后续任务按[闭环标准 v1](../../testing/verification-closed-loop-standard.md)留证;不再用“导入成功”“文件已生成”或历史分组通过数量单独宣布闭环完成。批量添加见[验收记录](../../testing/2026-11-18-verification-batch-import.md)。
- [x] C01~C05:开始前冻结名单/输入/策略,核对实际运行成员、预期结论、同源图表与报告、原文和退出证明;源文件不变、新增换料操作为0。
- [x] C06:实际包正常关闭后重启,证据可回读,尝试数不增加。
- [x] C07:实际包单项重校验,新attempt及generation递增,旧结果/报告不被覆盖。
- [x] C08受控回归:反脚、命名警告、必查未知、停止/资源未知、版本竞争和单项失败后继续的服务/Qt反例。
- [ ] C08实际包补充:原生导出失败项先运行,后续正常项仍完成;本轮正在执行并记录独立现场。
- [ ] C08实际包停止与资源未知:需要各自独立的受控现场,正常路径不替代。
- [ ] 真实已保存换料回执触发的自动交接实际包闭环;合同/幂等测试已通过,不能冒用独立输入证明。
- [ ] 连续默认全量稳定全绿;前轮既有时序波动仍交由原修复任务处理,不改旧断言。
单次只读验收入口为 `tools/verify_independent_verification.py --expect-verdict ...`,明确只覆盖 C01~C05;实际包入口为 `tools/accept_verification_closed_loop.py`,覆盖独立输入 C01~C07,并可用 `--fault-batch` 增加原生失败隔离反例。未测项在JSON里保留not_run。测试失败或标准不符均非零退出;已有换料历史采用执行前基线,不能要求历史总数为0。
用户授权的 N01~N12 已顺序实现、分步提交并完成本地开发包验收。新增组 81 项通过,实际包已验证独立导入、原生导出、查看证据、重校验和正常关闭。既有连续全量回归的两处时序波动仍保留,未合并、推送或发布。本分支没有复制另一修复任务的未提交修改。
实施进度及验证结果见 docs/testing/2026-11-17-independent-verification-acceptance.md。后续开发以本入口和[N01~N12清单](2026-11-17-independent-verification-new-features.md)共同为准,11月6日的V/U旧计划保留为历史参考,不能再并行照旧重建后端或来源界面。
@@ -0,0 +1,35 @@
# 工程校验闭环标准 v1
本标准用于 N12 及后续功能。写完代码、测试通过、工程检查一致、业务闭环通过、允许发布,是不同结论。每次验收必须声明正在证明哪一个结论。
## 单次功能闭环的通过条件
| 编号 | 必须证明 | 不满足时 |
| --- | --- | --- |
| C01 输入冻结 | 开始前写下预期工程名单、数量、前后摘要、输入版本和检查策略;配对预览与实际导入相符 | 失败;不能从运行后的结果反推预期 |
| C02 完整执行 | 用户显式开始;运行成员与冻结范围完全相同,每项都有属于本次运行的终态尝试 | 有排队、运行、停止、中断、漏项或重复项,均不能算完整闭环 |
| C03 判断正确 | 每项结论符合事先指定的预期;执行成功与 CONSISTENT/WARNING/DIFFERENT 分开;必查未知不得通过 | 误报、漏报、未知充当一致都失败 |
| C04 事实同源 | 页面、逐脚表、JSON和HTML报告绑定同一 item、输入版本、attempt、generation、输入摘要和证据摘要;原文摘要可复核 | 任何串版本、旧结果、缺证据或报告不符都失败 |
| C05 只读安全 | 校验前后全部已冻结源文件摘要不变,新增换料操作数为0,执行资源退出有证明;已有换料历史以开始前基线计数核对 | 不变性或退出证明缺失,失败并保留现场 |
| C06 重启回读 | 正常退出后重启,结果及证据仍可查看,尝试数不增加、不自动重跑 | 失败 |
| C07 重校验 | 同一输入重新执行时 attempt 更新、generation 递增;旧结果保留,新旧报告均可追溯 | 失败 |
| C08 异常隔离 | 反脚/断线等业务异常不挡后续工程;停止后未派发项仍待校验;资源/存储未知暂停正确范围;仍可新建独立任务 | 相应异常用例失败;不能以“正常样本跑过”替代 |
只有本次要求的检查全部通过,才写“该功能闭环通过”。验证程序对失败必须非零退出;结构性检查失败不能被 WARNING、正常进程退出或成功生成文件覆盖。某项故障测试验证了预期失败,不等于故障工程已经检查通过。
## 固定业务反例
物理脚1/2互换、断线、短接、位号或物理脚号变化必须为 DIFFERENT;loop→outloop 且连接不变必须为 WARNING;必查字段缺失必须为 INCOMPLETE。图中固定物理脚号位置,异常端点可与逐脚表和原文对应。自建格式夹具用于反例和可复现测试,必须明确标注,不能当成真实CAD导出证据。
## 两条业务路径分别验收
1. 独立输入:实际开发包中批量添加 → 开始全部 → 原生导出 → 查看图表 → 导出报告 → 正常关闭 → 重启查看 → 重校验 → 再导出。
2. 成功结果交接:真正已落盘的成功换料回执 → 自动登记校验 → 同一套执行、复核与报告。重复接收不重复建项,未保存项不得作为成功项。
两条路径不能互相代替。受控服务测试可证明交接合同、幂等和非阻塞;没有取得实际成功回执触发的包内证据时,该路径记“实际闭环未测”。本任务不为补验收重新执行客户换料,也不修改其他任务的旧执行器。
## 留证与发布
验收记录至少包含:代码提交、EXE SHA256、证据类型、预期清单、输入和报告摘要、任务/运行/尝试身份、每项标准的 passed/failed/not_run、失败原因、截图和实际命令。前一次运行、仅有截图、历史分组测试覆盖均不能替代本次闭环。
发布还要求相关回归通过、连续默认全量无未解决失败、实际包身份与源码相符、声明的两条实际路径通过。现有时序波动仍是发布保留项,不在本功能中改断言掩盖。引脚级Value和确切库内脚到焊盘映射仍未覆盖;不能写成“封装全部正确”。
@@ -76,7 +76,11 @@ def item_html(result):
def render_item_report(result_ref,destination):
destination=Path(destination)
if destination.exists():fail('报告文件已存在,请换一个名称。')
html='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>工程校验报告</title><style>'+STYLE+'</style><main>'+item_html(result_ref)+'</main></html>'
fields={'item-id':result_ref.get('item_id'),'attempt-id':result_ref.get('attempt_id'),
'input-version':result_ref.get('input_version'),'generation':result_ref.get('generation'),
'input-digest':result_ref.get('input_digest'),'evidence-digest':(result_ref.get('evidence') or {}).get('digest')}
binding=' '.join(f'data-{key}="{text(value)}"' for key,value in fields.items())
html='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>工程校验报告</title><style>'+STYLE+'</style><main '+binding+'>'+item_html(result_ref)+'</main></html>'
destination.parent.mkdir(parents=True,exist_ok=True)
temporary=destination.with_name(destination.name+'.'+uuid4().hex+'.tmp')
try:
+114
View File
@@ -0,0 +1,114 @@
import copy
import json
from pathlib import Path
import pytest
from test_independent_verification_intake import service
from test_independent_verification_runner import imported
from test_native_cli_netlist import graph
from test_verification_control import wait_idle
def complete(service, tmp_path, monkeypatch):
from tools.verification_closed_loop import freeze_inputs
from dx_productivity.trial_task_center import netlist_verification
task, item = imported(service, tmp_path)
scope = freeze_inputs(service, task, expected_count=1)
def native(capture, output, *, stop_event):
output.mkdir(parents=True, exist_ok=True)
return {'status': 'success', 'complete': True, 'capture': capture, 'graph': graph(output),
'outputs': {'effective': str(output/'effective.txt'), 'keyin': str(output/'design.kyn')}}
monkeypatch.setattr(netlist_verification, '_export_saved_capture', native)
run = service.start(task, request_id='loop-start'); wait_idle(service)
report = service.export_task(task, tmp_path/'reports', request_id='loop-report')
return task, item, scope, run, report
def test_complete_scope_report_and_inputs_pass_same_attempt_checks(service, tmp_path, monkeypatch):
from tools.verification_closed_loop import verify_closed_loop
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})
assert result['passed'] and all(c['state']=='passed' for c in result['checks'])
def test_empty_task_cannot_be_frozen_as_a_closed_loop(service):
from tools.verification_closed_loop import freeze_inputs
task = service.create_task('空', request_id='empty')['task_id']
with pytest.raises(ValueError): freeze_inputs(service, task, expected_count=0)
@pytest.mark.parametrize('fault', ['run_member', 'report_attempt', 'html_binding', 'source', 'expected_verdict', 'resource_proof', 'evidence_binding'])
def test_closed_loop_rejects_wrong_members_stale_reports_and_changed_sources(service, tmp_path, monkeypatch, fault):
from tools.verification_closed_loop import verify_closed_loop
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
expected = {item: 'CONSISTENT'}
if fault=='run_member':
run = copy.deepcopy(run); run['members'] = []
elif fault=='report_attempt':
path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8'))
value['items'][0]['attempt_id'] = 'old-attempt'
path.write_text(json.dumps(value), encoding='utf-8')
elif fault=='html_binding':
path = report/f'{item}-v1.html'; path.write_text('<html>CONSISTENT</html>', encoding='utf-8')
elif fault=='source':
capture = service.artifacts.read(scope['items'][0]['before_ref'], 'snapshot')
Path(capture['project']).write_text('changed', encoding='utf-8')
elif fault=='expected_verdict':
expected[item] = 'DIFFERENT'
elif fault=='resource_proof':
attempt = service.repo.latest_attempt(item, 1)
(service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').write_text(
json.dumps({'owner_id': 'another-owner', 'resource_state': 'released'}), encoding='utf-8')
else:
path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8'))
value['items'][0]['attempt']['result']['evidence']['binding']['input_digest'] = 'f'*64
path.write_text(json.dumps(value), encoding='utf-8')
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts=expected)
assert not result['passed'] and any(c['state']=='failed' for c in result['checks'])
def test_stopped_attempt_is_not_a_completed_closed_loop(service, tmp_path):
from tools.verification_closed_loop import freeze_inputs, verify_closed_loop
task, item = imported(service, tmp_path)
scope = freeze_inputs(service, task, expected_count=1)
attempt = service.repo.create_attempt(item, 1, request_id='cancelled')
service.repo.finish(attempt['attempt_id'], generation=attempt['generation'], result={
'attempt_state': 'cancelled', 'resource_state': 'released', 'verdict': 'INCOMPLETE', 'reason': 'stop'})
run = {'run_id': None, 'members': [{'item_id': item, 'input_version': 1, 'attempt_id': attempt['attempt_id']}]}
report = service.export_task(task, tmp_path/'reports', request_id='cancelled-report')
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'INCOMPLETE'})
assert not result['passed']
def test_selected_rerun_is_checked_against_its_scope_inside_a_whole_task_report(service, tmp_path, monkeypatch):
from tools.verification_closed_loop import freeze_inputs, verify_closed_loop
task, item, _, _, _ = complete(service, tmp_path, monkeypatch)
payload = {key: value for key, value in service.repo.revision(item, 1)['payload'].items()
if key not in ('payload_digest', 'input_state')}
payload.update(item_key='second', display_name='second')
service.publish_inputs(task, (payload,), submission_id='second')
scope = freeze_inputs(service, task, expected_count=2)
whole = service.start(task, selection=tuple((row['item_id'], 1) for row in scope['items']), request_id='whole-task'); wait_idle(service)
subset = dict(scope, items=[row for row in scope['items'] if row['item_id']==item])
selected_run = dict(whole, members=[row for row in whole['members'] if row['item_id']==item])
whole_report = service.export_task(task, tmp_path/'whole-report', request_id='whole-report')
assert verify_closed_loop(service, task, subset, selected_run, whole_report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True)['passed']
run = service.rerun(item, 1, request_id='selected'); wait_idle(service)
scope['items'] = [row for row in scope['items'] if row['item_id']==item]
report = service.export_task(task, tmp_path/'selected-report', request_id='selected-report')
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True)
assert result['passed']
assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
def test_existing_replacement_history_is_allowed_but_new_replacement_is_rejected(service, tmp_path, monkeypatch):
from tools.verification_closed_loop import verify_closed_loop
def operation(key):
with service.repo.store.transaction():
service.repo.connection.execute('''INSERT INTO execution_operations
(task_id,round_id,operation_key,command_json,state) VALUES ('prior','round',?,'{}','verified')''', (key,))
operation('before-verification')
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
assert verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
operation('during-verification')
assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
+232
View File
@@ -0,0 +1,232 @@
"""Actual Windows EXE: batch import, native run, report, restart, rerun, report.
Use saved test copies. Expectations must be supplied before launch. This tool
never creates replacement receipts or changes task state directly in SQLite.
"""
import argparse
from contextlib import contextmanager
import json
import os
import shutil
from pathlib import Path
import sqlite3
import subprocess
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'src'))
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.verification.service import VerificationService
from dx_productivity.trial_task_center.native_netlist import _hash
from verification_closed_loop import freeze_inputs, verify_closed_loop, require
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
for name in ('exe', 'before-root', 'after-root', 'output'):
parser.add_argument('--'+name, type=Path, required=True)
parser.add_argument('--expected-items', nargs='+', required=True, help='Exact relative paths, e.g. A/input.prj B/input.prj')
parser.add_argument('--expect-verdict', choices=('CONSISTENT', 'WARNING', 'DIFFERENT'), required=True)
parser.add_argument('--fault-batch', action='store_true', help='Use private copies and prepend one deliberately invalid database fixture')
args = parser.parse_args(argv)
output = args.output.resolve(); exe = args.exe.resolve()
before, after = args.before_root.resolve(), args.after_root.resolve()
if output.exists(): parser.error('output must be a new directory')
if output.is_relative_to(before) or output.is_relative_to(after): parser.error('output must be outside input roots')
if len(set(args.expected_items))!=len(args.expected_items): parser.error('expected-items must be unique')
output.mkdir(parents=True)
# Import UI automation only for the EXE path; service acceptance remains portable.
import win32con
import win32gui
from packaged_window_driver import WindowDriver, window_image
from accept_packaged_ui import wait
data = output/'data'
report = {'standard': 'verification-closed-loop-v1', 'validation_scope': 'independent-native:C01-C07',
'evidence_kind': 'native-packaged-ui', 'executable': str(exe), 'executable_sha256': _hash(exe),
'expected_items': args.expected_items, 'expected_verdict': args.expect_verdict, 'checks': [],
'not_run': ['C08:packaged-fault-cases', 'actual-successful-replacement-handoff', 'continuous-full-regression']}
original = {str(p): _hash(p) for root in (before, after) for p in root.rglob('*') if p.is_file()}
failed_name = '00-invalid/input.prj' if args.fault_batch else None
if args.fault_batch:
for phase, source in (('before', before), ('after', after)):
destination = output/'inputs'/phase
shutil.copytree(source, destination)
fixture = destination/'00-invalid'; (fixture/'database').mkdir(parents=True)
(fixture/'input.prj').write_text('KEY RootBlock "board"\nKEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n', encoding='utf-8')
(fixture/'database/icdb.dat').write_bytes(b'deliberately-invalid-acceptance-fixture')
before, after = output/'inputs/before', output/'inputs/after'
original.update((str(p), _hash(p)) for root in (before, after) for p in root.rglob('*') if p.is_file())
report.update(evidence_kind='native-real-copies-and-invalid-database-fixture', expected_failed_items=[failed_name])
expected_names = sorted(args.expected_items + ([failed_name] if failed_name else []))
process = driver = main_window = None
def sql(query, params=()):
with sqlite3.connect(data/'task-center.db') as db:
db.row_factory = sqlite3.Row
return [dict(row) for row in db.execute(query, params)]
@contextmanager
def reader():
store = TaskStore(data/'task-center.db')
try: yield VerificationService(store, data)
finally: store.close()
def launch():
nonlocal process, driver, main_window
environment = dict(os.environ)
for name in ('PYTHONPATH', 'PYTHONHOME', 'DX_TASK_CENTER_HOME', 'QT_QPA_PLATFORM'): environment.pop(name, None)
p = subprocess.Popen([str(exe), '--data-root', str(data)], cwd=exe.parent, env=environment,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=subprocess.CREATE_NO_WINDOW)
process = p
d = WindowDriver(p.pid, output); driver = d
def ready():
try: return d.main()
except TimeoutError:
require(p.poll() is None, 'Application exited before its main window appeared')
return None
window = wait(ready, timeout=90); main_window = window
d.click(d.find(automation_id='verification_navigation'))
return p, d, window
def close():
for window in driver.windows():
if win32gui.GetClassName(window.handle)=='#32770':
cancel = next((c for c in window.descendants(control_type='Button') if c.element_info.automation_id=='2'), None)
if cancel: win32gui.PostMessage(cancel.handle, win32con.BM_CLICK, 0, 0)
win32gui.PostMessage(main_window.handle, win32con.WM_CLOSE, 0, 0)
require(process.wait(45)==0, 'Application did not exit normally')
def run_record(run_id):
row = sql('SELECT * FROM verification_runs WHERE run_id=?', (run_id,))[0]
require(row['state']=='completed', 'Run stopped, blocked or interrupted')
return {'run_id': run_id, 'members': json.loads(row['members_json'])}
def finished(run_id):
rows = sql('SELECT state FROM verification_runs WHERE run_id=?', (run_id,))
return rows and rows[0]['state'] not in ('queued', 'running', 'stopping')
def export_report():
destination = output/'reports'; destination.mkdir(exist_ok=True)
previous = set(destination.glob('verification-*'))
driver.click(driver.find('导出报告', kind='Button'))
dialog = wait(lambda: next((w for w in driver.windows() if win32gui.GetClassName(w.handle)=='#32770'), None), timeout=15)
# The native folder picker has FolderName edit 1152, unlike file picker 1001.
field = next(c for c in dialog.descendants(control_type='Edit') if c.element_info.automation_id=='1152')
field.iface_value.SetValue(str(destination))
require(field.get_value()==str(destination), 'Folder picker path differs from the requested destination')
confirm = next(c for c in dialog.descendants(control_type='Button') if c.element_info.automation_id=='1')
win32gui.PostMessage(confirm.handle, win32con.BM_CLICK, 0, 0)
wait(lambda: not win32gui.IsWindow(dialog.handle), timeout=20)
return wait(lambda: next((p for p in destination.glob('verification-*') if p not in previous and (p/'task.json').is_file()), None), timeout=60)
def selected_cell(name):
tables = [w for kind in ('Table', 'DataGrid') for w in main_window.descendants(control_type=kind) if w.is_visible()]
table = min(tables, key=lambda w: w.rectangle().left)
return next((c for c in table.children() if c.window_text()==name), None)
def verify_report(scope, run, directory, label, *, selected_only=False):
with reader() as service:
validation = verify_closed_loop(service, task, scope, run, directory,
expected_verdicts={row['item_id']: args.expect_verdict for row in scope['items']}, selected_only=selected_only)
(output/(label+'.json')).write_text(json.dumps(validation, ensure_ascii=False, indent=2), encoding='utf-8')
report['checks'].extend(validation['checks'])
require(validation['passed'], label+' failed: '+json.dumps(validation['checks'], ensure_ascii=False))
try:
process, driver, main_window = launch()
reduce = driver.find('减少透明度', kind='Button', optional=True)
if reduce: driver.click(reduce)
driver.click(wait(lambda: driver.find(automation_id='verificationNew')))
dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='新建校验'), None))
ok = next(b for b in dialog.descendants(control_type='Button') if b.window_text().replace('&', '') in ('OK', '确定'))
driver.click(ok)
button = wait(lambda: driver.find(automation_id='verificationBatchImport'))
wait(lambda: button.is_enabled()); driver.click(button)
dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='批量添加工程对'), None))
driver.type_text(driver.find(automation_id='verificationBeforeRoot', window=dialog), str(before))
driver.type_text(driver.find(automation_id='verificationAfterRoot', window=dialog), str(after))
driver.click(driver.find(automation_id='verificationScanPairs', window=dialog))
button = driver.find(automation_id='verificationImportPairs', window=dialog)
wait(lambda: button.is_enabled()); window_image(dialog.handle).save(output/'01-pairs.png')
driver.click(button)
wait(lambda: driver.find(f'本批导入完成:已导入 {len(expected_names)} 对,失败 0 对;尚未开始校验。', kind='Text', window=dialog), timeout=120)
driver.click(driver.find('关闭', kind='Button', window=dialog))
imported = sql('SELECT * FROM verification_items ORDER BY display_name')
require([r['display_name'] for r in imported]==expected_names, 'Imported names differ from the predeclared scope')
require(not sql('SELECT attempt_id FROM verification_attempts'), 'Import started verification automatically')
task = imported[0]['task_id']
with reader() as service: scope = freeze_inputs(service, task, expected_count=len(expected_names))
(output/'frozen-scope.json').write_text(json.dumps(dict(scope, expected_verdict=args.expect_verdict,
expected_failed_items=[failed_name] if failed_name else []), ensure_ascii=False, indent=2), encoding='utf-8')
driver.click(driver.find('开始全部', kind='Button'))
run_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs')), None))
wait(lambda: finished(run_id), timeout=900)
run = run_record(run_id)
if failed_name:
failed_item = next(r['item_id'] for r in imported if r['display_name']==failed_name)
with reader() as service:
failure = service.read_item(failed_item)
require(run['members'][0]['item_id']==failed_item, 'Fault fixture did not run before the healthy projects')
require(failure['attempt_state']=='failed' and failure['verdict']=='INCOMPLETE'
and failure['attempt']['result']['resource_state']=='released', 'Invalid database did not fail and release resources as expected')
report['checks'].append({'check_id': 'C08:failed-native-item-continues', 'state': 'passed',
'item_id': failed_item, 'observed_state': failure['attempt_state'], 'observed_verdict': failure['verdict']})
report['not_run'][0] = 'C08:packaged-stop-and-uncertain-resources'
scope = dict(scope, items=[row for row in scope['items'] if row['item_id']!=failed_item])
run = dict(run, members=[row for row in run['members'] if row['item_id']!=failed_item])
name = sorted(args.expected_items)[0]
driver.click(wait(lambda: selected_cell(name)))
label = {'CONSISTENT': '一致', 'WARNING': '需关注', 'DIFFERENT': '异常'}[args.expect_verdict]
wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60)
window_image(main_window.handle).save(output/'02-result.png')
directory = export_report(); verify_report(scope, run, directory, 'first-run', selected_only=bool(failed_name))
first_report_hashes = {str(p): _hash(p) for p in directory.iterdir() if p.is_file()}
attempts = sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id')
report['first_report'] = str(directory)
close(); process, driver, main_window = launch()
driver.click(wait(lambda: selected_cell(name)))
wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60)
# Wait for startup recovery and at least one normal workspace polling interval.
time.sleep(2)
require(sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id')==attempts, 'Restart changed attempts or started another run')
verify_report(scope, run, directory, 'restart-readback', selected_only=bool(failed_name))
report['checks'].append({'check_id': 'C06:restart', 'state': 'passed'})
window_image(main_window.handle).save(output/'03-restart.png')
old_ids = {r['run_id'] for r in sql('SELECT run_id FROM verification_runs')}
driver.click(driver.find('重校验', kind='Button'))
rerun_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs') if r['run_id'] not in old_ids), None))
wait(lambda: finished(rerun_id), timeout=900)
rerun = run_record(rerun_id)
subset = dict(scope, items=[r for r in scope['items'] if r['display_name']==name])
member = rerun['members'][0]
previous = next(r for r in attempts if r['attempt_id']==next(m['attempt_id'] for m in run['members'] if m['item_id']==member['item_id']))
latest = sql('SELECT * FROM verification_attempts WHERE attempt_id=?', (member['attempt_id'],))[0]
require(latest['generation']==previous['generation']+1 and latest['input_version']==previous['input_version'], 'Rerun did not create the next attempt on the same input')
require(all(_hash(Path(p))==sha for p, sha in first_report_hashes.items()), 'Rerun overwrote the first report')
require(sql('SELECT result_json FROM verification_attempts WHERE attempt_id=?', (previous['attempt_id'],))[0]['result_json']==previous['result_json'], 'Rerun changed the old result')
second_report = export_report()
verify_report(subset, rerun, second_report, 'rerun', selected_only=True)
report['checks'].append({'check_id': 'C07:rerun', 'state': 'passed'})
report['rerun_report'] = str(second_report)
require(all(_hash(Path(p))==sha for p, sha in original.items()), 'Input files changed during the loop')
require(sql('SELECT COUNT(*) total FROM execution_operations')[0]['total']==0, 'Replacement operations occurred')
close(); report['passed'] = True
except Exception as error:
report.update(passed=False, reason=str(error), error=type(error).__name__)
if driver:
for n, window in enumerate(driver.windows()):
try: window_image(window.handle).save(output/f'failure-{n}.png')
except Exception: pass
if process and process.poll() is None and main_window:
try: close()
except Exception: report['process_left_running'] = process.pid
elif process and process.poll() is None:
report['process_left_running'] = process.pid
finally:
report['input_sha256'] = original
(output/'acceptance.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(report, ensure_ascii=False))
return 0 if report.get('passed') else 1
if __name__=='__main__': raise SystemExit(main())
+144
View File
@@ -0,0 +1,144 @@
"""Independent acceptance checks against a scope recorded BEFORE execution."""
from html.parser import HTMLParser
import json
from pathlib import Path
from dx_productivity.trial_task_center.native_netlist import _hash
def require(condition, message):
if not condition:
raise ValueError(message)
def records(service, task_id):
result, cursor = [], None
while True:
page = service.read_task(task_id, cursor=cursor)
result.extend(service.read_item(row['item_id']) for row in page['items'])
cursor = page['cursor']
if cursor is None:
return result
def freeze_inputs(service, task_id, *, expected_count):
rows = records(service, task_id)
require(type(expected_count) is int and expected_count>0 and len(rows)==expected_count,
'验收必须事先指定非空、准确的工程数量。')
sources, items = {}, []
for row in rows:
require(row['input_state']=='ready', '开始前输入尚未就绪。')
payload = row['payload']
for name in ('before_ref', 'after_ref'):
capture = service.artifacts.read(payload[name], 'snapshot')
for path, sha in capture['source_hashes'].items():
require(path not in sources or sources[path]==sha, '同一源文件出现不同摘要。')
require(_hash(Path(path))==sha, '源文件已偏离导入时的版本。')
sources[path] = sha
items.append({'item_id': row['item_id'], 'input_version': row['input_version'],
'input_digest': payload['payload_digest'], 'before_ref': payload['before_ref'],
'after_ref': payload['after_ref'], 'policy': payload['check_plan'], 'display_name': row['display_name']})
operations = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]
return {'standard': 'verification-closed-loop-v1', 'task_id': task_id, 'items': items,
'source_hashes': sources, 'replacement_operations_before': operations}
class _ReportHTML(HTMLParser):
def __init__(self, text):
super().__init__(); self.mains = []; self.diagrams = 0
self.feed(text)
def handle_starttag(self, tag, attrs):
if tag=='main': self.mains.append(dict(attrs))
if tag=='svg': self.diagrams += 1
def verify_closed_loop(service, task_id, scope, run, report, *, expected_verdicts, selected_only=False):
checks = []
def check(name, action):
try:
action()
checks.append({'check_id': name, 'state': 'passed'})
except (ValueError, OSError, KeyError, TypeError) as error:
checks.append({'check_id': name, 'state': 'failed', 'reason': str(error)})
frozen = {r['item_id']: r for r in scope['items']}
members = {r['item_id']: r for r in run['members']}
report = Path(report)
current = {r['item_id']: r for r in records(service, task_id)}
if selected_only:
current = {key: row for key, row in current.items() if key in frozen}
def membership():
require(scope['task_id']==task_id and bool(frozen), '冻结任务为空或身份错误。')
require(len(frozen)==len(scope['items']) and len(members)==len(run['members']), '工程或运行成员重复。')
require(set(frozen)==set(members)==set(current)==set(expected_verdicts), '冻结范围、运行成员、当前工程或预期清单不一致。')
durable = service.repo.connection.execute('SELECT state,members_json FROM verification_runs WHERE run_id=?', (run['run_id'],)).fetchone()
require(durable is not None and durable['state']=='completed', '运行未完成。')
persisted = json.loads(durable['members_json'])
require(all(row in persisted for row in run['members']) if selected_only else persisted==run['members'],
'运行成员与持久记录不同。')
require(all(v in ('CONSISTENT', 'WARNING', 'DIFFERENT') for v in expected_verdicts.values()),
'正常完整闭环必须事先指定可核验的工程结论。')
check('C01-C02:scope', membership)
saved = {}
def report_scope():
value = json.loads((report/'task.json').read_text(encoding='utf-8'))
require(value['task_id']==task_id, '报告任务不符。')
identities = {r['item_id'] for r in value['items']}
require(len(identities)==len(value['items']) and (set(frozen)<=identities if selected_only else identities==set(frozen)), '报告工程范围不符。')
require((report/'index.html').is_file(), '缺少报告首页。')
saved.update((r['item_id'], r) for r in value['items'])
check('C04:report_scope', report_scope)
for item, original in frozen.items():
def item_result(item=item, original=original):
row, member = current[item], members[item]
require(row['input_version']==member['input_version']==original['input_version'], '输入版本不符。')
require(row['payload']['payload_digest']==original['input_digest'], '输入摘要变化。')
for key in ('before_ref', 'after_ref'):
require(row['payload'][key]==original[key], '冻结副本引用变化。')
attempt = row['attempt']
require(attempt is not None and attempt['state']=='succeeded', '尝试未成功完成,停止/失败不能算闭环。')
require(attempt['run_id']==run['run_id'] and row['attempt_id']==member['attempt_id'], '尝试不属于本次运行。')
service.validate_attempt(attempt)
result = attempt['result']; evidence = result['evidence']; binding = evidence['binding']
require(result['verdict']==expected_verdicts[item], '工程结论与预期不符。')
for key, expected in (('item_id', item), ('attempt_id', attempt['attempt_id']),
('generation', attempt['generation']), ('input_version', original['input_version']),
('input_digest', original['input_digest']), ('policy', original['policy'])):
require(result[key]==expected, '结果身份不符:'+key)
for key in ('attempt_id', 'input_version', 'input_digest'):
require(binding[key]==result[key], '图表证据绑定不符:'+key)
require(result['resource_state']=='released', '执行资源退出未确认。')
released = json.loads((service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').read_text(encoding='utf-8'))
require(released['resource_state']=='released' and released['owner_id']==attempt['owner_id'], '资源退出证明不属于本次尝试。')
if result['verdict'] in ('CONSISTENT', 'WARNING'):
states = {c['check_id']: c['state'] for c in result['checks']}
require(all(states.get(key) in ('equal', 'expected_change', 'warning')
for key in original['policy']['required_checks']), '必查项缺失或未知。')
else:
require(any(c['state']=='error' for c in result['checks']), '异常结论缺少错误事实。')
published = saved[item]
require(published['input_version']==original['input_version'] and published['attempt_id']==attempt['attempt_id'],
'报告引用旧输入或旧尝试。')
require(published['attempt']['result']==result, 'JSON报告事实与当前结果不同。')
html = _ReportHTML((report/f'{item}-v{original["input_version"]}.html').read_text(encoding='utf-8'))
require(len(html.mains)==1, 'HTML报告缺少唯一绑定。')
for key, value in (('item-id', item), ('attempt-id', attempt['attempt_id']), ('generation', attempt['generation']),
('input-version', original['input_version']), ('input-digest', original['input_digest']),
('evidence-digest', evidence['digest'])):
require(html.mains[0].get('data-'+key)==str(value), 'HTML报告绑定不符:'+key)
require(not evidence['pins'] or html.diagrams>0, '有逐脚事实却没有报告接线图。')
check('C02-C04:'+item, item_result)
def unchanged():
require(bool(scope['source_hashes']), '没有冻结源文件摘要。')
require(all(_hash(Path(path))==sha for path, sha in scope['source_hashes'].items()), '源文件被改写。')
count = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]
# Early v1 standalone fixtures recorded an implicit zero baseline.
require(count==scope.get('replacement_operations_before', 0), '校验期间出现了新的换料操作或历史记录变化。')
check('C05:read_only', unchanged)
return {'standard': scope['standard'], 'task_id': task_id, 'run_id': run['run_id'],
'validation_scope': 'single-run:C01-C05', 'not_run': ['C06', 'C07', 'C08'],
'passed': all(c['state']=='passed' for c in checks), 'checks': checks}
+11 -3
View File
@@ -12,6 +12,7 @@ from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.verification.service import VerificationService
from dx_productivity.trial_task_center import native_netlist,netlist_verification
from dx_productivity.trial_task_center.batch_resources import write_record
from verification_closed_loop import freeze_inputs,verify_closed_loop
def main(argv=None):
@@ -20,6 +21,8 @@ def main(argv=None):
group.add_argument('--fixture-root',type=Path)
group.add_argument('--native-before',type=Path)
parser.add_argument('--native-after',type=Path);parser.add_argument('--output',type=Path,required=True)
parser.add_argument('--expect-verdict',choices=('CONSISTENT','WARNING','DIFFERENT'),default='CONSISTENT',
help='Expected conclusion fixed before running; defaults to CONSISTENT')
args=parser.parse_args(argv)
if args.native_before and not args.native_after:parser.error('--native-after is required')
root=args.output.resolve()
@@ -48,6 +51,9 @@ def main(argv=None):
task=service.create_task('独立校验验收',request_id='acceptance-create')['task_id']
accepted=service.import_pair(task,before,after,request_id='acceptance-import',stop_event=Event())
if not accepted['accepted']:raise ValueError('No input accepted')
frozen=freeze_inputs(service,task,expected_count=1)
expectations={row['item_id']:args.expect_verdict for row in frozen['items']}
write_record(root/'frozen-scope.json',dict(frozen,expected_verdicts=expectations,evidence_kind=evidence_kind))
run=service.start(task,request_id='acceptance-run')
deadline=time.monotonic()+900
while service.busy:
@@ -55,13 +61,15 @@ def main(argv=None):
time.sleep(.05)
current=service.read_task(task);report=service.export_task(task,root/'reports',request_id='acceptance-report')
unchanged=all(native_netlist._hash(Path(p))==sha for p,sha in initial.items())
closed_loop=verify_closed_loop(service,task,frozen,run,report,expected_verdicts=expectations)
write_record(root/'closed-loop.json',closed_loop)
summary={'evidence_kind':evidence_kind,'task_id':task,'run':run,'summary':current['summary'],
'input_sha256':initial,'inputs_unchanged':unchanged,'report':str(report),
'policy':'verification-v1','replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]}
'policy':'verification-v1','closed_loop':closed_loop,
'replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]}
write_record(root/'acceptance.json',summary);print(json.dumps(summary,ensure_ascii=False))
if not unchanged:return 4
if current['summary']['different']:return 2
if current['summary']['unavailable'] or current['summary']['waiting'] or current['summary']['running']:return 3
if not closed_loop['passed']:return 3
return 0
except Exception as error:
write_record(root/'failure.json',{'error':type(error).__name__,'reason':str(error)})