test: enforce reproducible verification closed-loop standards
This commit is contained in:
1 parent
a74b810786
commit
e65bed0408
7 files changed
+556
-4
No files matched your search
@@ -138,6 +138,21 @@ input_record = {
|
||||
|
||||
## 8. 交接状态
|
||||
|
||||
### N12 后续闭环标准(2026-11-18)
|
||||
|
||||
所有后续任务按[闭环标准 v1](../../testing/verification-closed-loop-standard.md)留证;不再用“导入成功”“文件已生成”或历史分组通过数量单独宣布闭环完成。批量添加见[验收记录](../../testing/2026-11-18-verification-batch-import.md)。
|
||||
|
||||
- [x] C01~C05:开始前冻结名单/输入/策略,核对实际运行成员、预期结论、同源图表与报告、原文和退出证明;源文件不变、新增换料操作为0。
|
||||
- [x] C06:实际包正常关闭后重启,证据可回读,尝试数不增加。
|
||||
- [x] C07:实际包单项重校验,新attempt及generation递增,旧结果/报告不被覆盖。
|
||||
- [x] C08受控回归:反脚、命名警告、必查未知、停止/资源未知、版本竞争和单项失败后继续的服务/Qt反例。
|
||||
- [ ] C08实际包补充:原生导出失败项先运行,后续正常项仍完成;本轮正在执行并记录独立现场。
|
||||
- [ ] C08实际包停止与资源未知:需要各自独立的受控现场,正常路径不替代。
|
||||
- [ ] 真实已保存换料回执触发的自动交接实际包闭环;合同/幂等测试已通过,不能冒用独立输入证明。
|
||||
- [ ] 连续默认全量稳定全绿;前轮既有时序波动仍交由原修复任务处理,不改旧断言。
|
||||
|
||||
单次只读验收入口为 `tools/verify_independent_verification.py --expect-verdict ...`,明确只覆盖 C01~C05;实际包入口为 `tools/accept_verification_closed_loop.py`,覆盖独立输入 C01~C07,并可用 `--fault-batch` 增加原生失败隔离反例。未测项在JSON里保留not_run。测试失败或标准不符均非零退出;已有换料历史采用执行前基线,不能要求历史总数为0。
|
||||
|
||||
用户授权的 N01~N12 已顺序实现、分步提交并完成本地开发包验收。新增组 81 项通过,实际包已验证独立导入、原生导出、查看证据、重校验和正常关闭。既有连续全量回归的两处时序波动仍保留,未合并、推送或发布。本分支没有复制另一修复任务的未提交修改。
|
||||
|
||||
实施进度及验证结果见 docs/testing/2026-11-17-independent-verification-acceptance.md。后续开发以本入口和[N01~N12清单](2026-11-17-independent-verification-new-features.md)共同为准,11月6日的V/U旧计划保留为历史参考,不能再并行照旧重建后端或来源界面。
|
||||
@@ -0,0 +1,35 @@
|
||||
# 工程校验闭环标准 v1
|
||||
|
||||
本标准用于 N12 及后续功能。写完代码、测试通过、工程检查一致、业务闭环通过、允许发布,是不同结论。每次验收必须声明正在证明哪一个结论。
|
||||
|
||||
## 单次功能闭环的通过条件
|
||||
|
||||
| 编号 | 必须证明 | 不满足时 |
|
||||
| --- | --- | --- |
|
||||
| C01 输入冻结 | 开始前写下预期工程名单、数量、前后摘要、输入版本和检查策略;配对预览与实际导入相符 | 失败;不能从运行后的结果反推预期 |
|
||||
| C02 完整执行 | 用户显式开始;运行成员与冻结范围完全相同,每项都有属于本次运行的终态尝试 | 有排队、运行、停止、中断、漏项或重复项,均不能算完整闭环 |
|
||||
| C03 判断正确 | 每项结论符合事先指定的预期;执行成功与 CONSISTENT/WARNING/DIFFERENT 分开;必查未知不得通过 | 误报、漏报、未知充当一致都失败 |
|
||||
| C04 事实同源 | 页面、逐脚表、JSON和HTML报告绑定同一 item、输入版本、attempt、generation、输入摘要和证据摘要;原文摘要可复核 | 任何串版本、旧结果、缺证据或报告不符都失败 |
|
||||
| C05 只读安全 | 校验前后全部已冻结源文件摘要不变,新增换料操作数为0,执行资源退出有证明;已有换料历史以开始前基线计数核对 | 不变性或退出证明缺失,失败并保留现场 |
|
||||
| C06 重启回读 | 正常退出后重启,结果及证据仍可查看,尝试数不增加、不自动重跑 | 失败 |
|
||||
| C07 重校验 | 同一输入重新执行时 attempt 更新、generation 递增;旧结果保留,新旧报告均可追溯 | 失败 |
|
||||
| C08 异常隔离 | 反脚/断线等业务异常不挡后续工程;停止后未派发项仍待校验;资源/存储未知暂停正确范围;仍可新建独立任务 | 相应异常用例失败;不能以“正常样本跑过”替代 |
|
||||
|
||||
只有本次要求的检查全部通过,才写“该功能闭环通过”。验证程序对失败必须非零退出;结构性检查失败不能被 WARNING、正常进程退出或成功生成文件覆盖。某项故障测试验证了预期失败,不等于故障工程已经检查通过。
|
||||
|
||||
## 固定业务反例
|
||||
|
||||
物理脚1/2互换、断线、短接、位号或物理脚号变化必须为 DIFFERENT;loop→outloop 且连接不变必须为 WARNING;必查字段缺失必须为 INCOMPLETE。图中固定物理脚号位置,异常端点可与逐脚表和原文对应。自建格式夹具用于反例和可复现测试,必须明确标注,不能当成真实CAD导出证据。
|
||||
|
||||
## 两条业务路径分别验收
|
||||
|
||||
1. 独立输入:实际开发包中批量添加 → 开始全部 → 原生导出 → 查看图表 → 导出报告 → 正常关闭 → 重启查看 → 重校验 → 再导出。
|
||||
2. 成功结果交接:真正已落盘的成功换料回执 → 自动登记校验 → 同一套执行、复核与报告。重复接收不重复建项,未保存项不得作为成功项。
|
||||
|
||||
两条路径不能互相代替。受控服务测试可证明交接合同、幂等和非阻塞;没有取得实际成功回执触发的包内证据时,该路径记“实际闭环未测”。本任务不为补验收重新执行客户换料,也不修改其他任务的旧执行器。
|
||||
|
||||
## 留证与发布
|
||||
|
||||
验收记录至少包含:代码提交、EXE SHA256、证据类型、预期清单、输入和报告摘要、任务/运行/尝试身份、每项标准的 passed/failed/not_run、失败原因、截图和实际命令。前一次运行、仅有截图、历史分组测试覆盖均不能替代本次闭环。
|
||||
|
||||
发布还要求相关回归通过、连续默认全量无未解决失败、实际包身份与源码相符、声明的两条实际路径通过。现有时序波动仍是发布保留项,不在本功能中改断言掩盖。引脚级Value和确切库内脚到焊盘映射仍未覆盖;不能写成“封装全部正确”。
|
||||
@@ -76,7 +76,11 @@ def item_html(result):
|
||||
def render_item_report(result_ref,destination):
|
||||
destination=Path(destination)
|
||||
if destination.exists():fail('报告文件已存在,请换一个名称。')
|
||||
html='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>工程校验报告</title><style>'+STYLE+'</style><main>'+item_html(result_ref)+'</main></html>'
|
||||
fields={'item-id':result_ref.get('item_id'),'attempt-id':result_ref.get('attempt_id'),
|
||||
'input-version':result_ref.get('input_version'),'generation':result_ref.get('generation'),
|
||||
'input-digest':result_ref.get('input_digest'),'evidence-digest':(result_ref.get('evidence') or {}).get('digest')}
|
||||
binding=' '.join(f'data-{key}="{text(value)}"' for key,value in fields.items())
|
||||
html='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>工程校验报告</title><style>'+STYLE+'</style><main '+binding+'>'+item_html(result_ref)+'</main></html>'
|
||||
destination.parent.mkdir(parents=True,exist_ok=True)
|
||||
temporary=destination.with_name(destination.name+'.'+uuid4().hex+'.tmp')
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from test_independent_verification_intake import service
|
||||
from test_independent_verification_runner import imported
|
||||
from test_native_cli_netlist import graph
|
||||
from test_verification_control import wait_idle
|
||||
|
||||
|
||||
def complete(service, tmp_path, monkeypatch):
|
||||
from tools.verification_closed_loop import freeze_inputs
|
||||
from dx_productivity.trial_task_center import netlist_verification
|
||||
task, item = imported(service, tmp_path)
|
||||
scope = freeze_inputs(service, task, expected_count=1)
|
||||
def native(capture, output, *, stop_event):
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
return {'status': 'success', 'complete': True, 'capture': capture, 'graph': graph(output),
|
||||
'outputs': {'effective': str(output/'effective.txt'), 'keyin': str(output/'design.kyn')}}
|
||||
monkeypatch.setattr(netlist_verification, '_export_saved_capture', native)
|
||||
run = service.start(task, request_id='loop-start'); wait_idle(service)
|
||||
report = service.export_task(task, tmp_path/'reports', request_id='loop-report')
|
||||
return task, item, scope, run, report
|
||||
|
||||
|
||||
def test_complete_scope_report_and_inputs_pass_same_attempt_checks(service, tmp_path, monkeypatch):
|
||||
from tools.verification_closed_loop import verify_closed_loop
|
||||
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
|
||||
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})
|
||||
assert result['passed'] and all(c['state']=='passed' for c in result['checks'])
|
||||
|
||||
|
||||
def test_empty_task_cannot_be_frozen_as_a_closed_loop(service):
|
||||
from tools.verification_closed_loop import freeze_inputs
|
||||
task = service.create_task('空', request_id='empty')['task_id']
|
||||
with pytest.raises(ValueError): freeze_inputs(service, task, expected_count=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fault', ['run_member', 'report_attempt', 'html_binding', 'source', 'expected_verdict', 'resource_proof', 'evidence_binding'])
|
||||
def test_closed_loop_rejects_wrong_members_stale_reports_and_changed_sources(service, tmp_path, monkeypatch, fault):
|
||||
from tools.verification_closed_loop import verify_closed_loop
|
||||
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
|
||||
expected = {item: 'CONSISTENT'}
|
||||
if fault=='run_member':
|
||||
run = copy.deepcopy(run); run['members'] = []
|
||||
elif fault=='report_attempt':
|
||||
path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8'))
|
||||
value['items'][0]['attempt_id'] = 'old-attempt'
|
||||
path.write_text(json.dumps(value), encoding='utf-8')
|
||||
elif fault=='html_binding':
|
||||
path = report/f'{item}-v1.html'; path.write_text('<html>CONSISTENT</html>', encoding='utf-8')
|
||||
elif fault=='source':
|
||||
capture = service.artifacts.read(scope['items'][0]['before_ref'], 'snapshot')
|
||||
Path(capture['project']).write_text('changed', encoding='utf-8')
|
||||
elif fault=='expected_verdict':
|
||||
expected[item] = 'DIFFERENT'
|
||||
elif fault=='resource_proof':
|
||||
attempt = service.repo.latest_attempt(item, 1)
|
||||
(service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').write_text(
|
||||
json.dumps({'owner_id': 'another-owner', 'resource_state': 'released'}), encoding='utf-8')
|
||||
else:
|
||||
path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8'))
|
||||
value['items'][0]['attempt']['result']['evidence']['binding']['input_digest'] = 'f'*64
|
||||
path.write_text(json.dumps(value), encoding='utf-8')
|
||||
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts=expected)
|
||||
assert not result['passed'] and any(c['state']=='failed' for c in result['checks'])
|
||||
|
||||
|
||||
def test_stopped_attempt_is_not_a_completed_closed_loop(service, tmp_path):
|
||||
from tools.verification_closed_loop import freeze_inputs, verify_closed_loop
|
||||
task, item = imported(service, tmp_path)
|
||||
scope = freeze_inputs(service, task, expected_count=1)
|
||||
attempt = service.repo.create_attempt(item, 1, request_id='cancelled')
|
||||
service.repo.finish(attempt['attempt_id'], generation=attempt['generation'], result={
|
||||
'attempt_state': 'cancelled', 'resource_state': 'released', 'verdict': 'INCOMPLETE', 'reason': 'stop'})
|
||||
run = {'run_id': None, 'members': [{'item_id': item, 'input_version': 1, 'attempt_id': attempt['attempt_id']}]}
|
||||
report = service.export_task(task, tmp_path/'reports', request_id='cancelled-report')
|
||||
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'INCOMPLETE'})
|
||||
assert not result['passed']
|
||||
|
||||
|
||||
def test_selected_rerun_is_checked_against_its_scope_inside_a_whole_task_report(service, tmp_path, monkeypatch):
|
||||
from tools.verification_closed_loop import freeze_inputs, verify_closed_loop
|
||||
task, item, _, _, _ = complete(service, tmp_path, monkeypatch)
|
||||
payload = {key: value for key, value in service.repo.revision(item, 1)['payload'].items()
|
||||
if key not in ('payload_digest', 'input_state')}
|
||||
payload.update(item_key='second', display_name='second')
|
||||
service.publish_inputs(task, (payload,), submission_id='second')
|
||||
scope = freeze_inputs(service, task, expected_count=2)
|
||||
whole = service.start(task, selection=tuple((row['item_id'], 1) for row in scope['items']), request_id='whole-task'); wait_idle(service)
|
||||
subset = dict(scope, items=[row for row in scope['items'] if row['item_id']==item])
|
||||
selected_run = dict(whole, members=[row for row in whole['members'] if row['item_id']==item])
|
||||
whole_report = service.export_task(task, tmp_path/'whole-report', request_id='whole-report')
|
||||
assert verify_closed_loop(service, task, subset, selected_run, whole_report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True)['passed']
|
||||
run = service.rerun(item, 1, request_id='selected'); wait_idle(service)
|
||||
scope['items'] = [row for row in scope['items'] if row['item_id']==item]
|
||||
report = service.export_task(task, tmp_path/'selected-report', request_id='selected-report')
|
||||
result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True)
|
||||
assert result['passed']
|
||||
assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
|
||||
|
||||
|
||||
def test_existing_replacement_history_is_allowed_but_new_replacement_is_rejected(service, tmp_path, monkeypatch):
|
||||
from tools.verification_closed_loop import verify_closed_loop
|
||||
def operation(key):
|
||||
with service.repo.store.transaction():
|
||||
service.repo.connection.execute('''INSERT INTO execution_operations
|
||||
(task_id,round_id,operation_key,command_json,state) VALUES ('prior','round',?,'{}','verified')''', (key,))
|
||||
operation('before-verification')
|
||||
task, item, scope, run, report = complete(service, tmp_path, monkeypatch)
|
||||
assert verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
|
||||
operation('during-verification')
|
||||
assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed']
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Actual Windows EXE: batch import, native run, report, restart, rerun, report.
|
||||
|
||||
Use saved test copies. Expectations must be supplied before launch. This tool
|
||||
never creates replacement receipts or changes task state directly in SQLite.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'src'))
|
||||
from dx_productivity.trial_task_center.store import TaskStore
|
||||
from dx_productivity.trial_task_center.verification.service import VerificationService
|
||||
from dx_productivity.trial_task_center.native_netlist import _hash
|
||||
from verification_closed_loop import freeze_inputs, verify_closed_loop, require
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ('exe', 'before-root', 'after-root', 'output'):
|
||||
parser.add_argument('--'+name, type=Path, required=True)
|
||||
parser.add_argument('--expected-items', nargs='+', required=True, help='Exact relative paths, e.g. A/input.prj B/input.prj')
|
||||
parser.add_argument('--expect-verdict', choices=('CONSISTENT', 'WARNING', 'DIFFERENT'), required=True)
|
||||
parser.add_argument('--fault-batch', action='store_true', help='Use private copies and prepend one deliberately invalid database fixture')
|
||||
args = parser.parse_args(argv)
|
||||
output = args.output.resolve(); exe = args.exe.resolve()
|
||||
before, after = args.before_root.resolve(), args.after_root.resolve()
|
||||
if output.exists(): parser.error('output must be a new directory')
|
||||
if output.is_relative_to(before) or output.is_relative_to(after): parser.error('output must be outside input roots')
|
||||
if len(set(args.expected_items))!=len(args.expected_items): parser.error('expected-items must be unique')
|
||||
output.mkdir(parents=True)
|
||||
# Import UI automation only for the EXE path; service acceptance remains portable.
|
||||
import win32con
|
||||
import win32gui
|
||||
from packaged_window_driver import WindowDriver, window_image
|
||||
from accept_packaged_ui import wait
|
||||
data = output/'data'
|
||||
report = {'standard': 'verification-closed-loop-v1', 'validation_scope': 'independent-native:C01-C07',
|
||||
'evidence_kind': 'native-packaged-ui', 'executable': str(exe), 'executable_sha256': _hash(exe),
|
||||
'expected_items': args.expected_items, 'expected_verdict': args.expect_verdict, 'checks': [],
|
||||
'not_run': ['C08:packaged-fault-cases', 'actual-successful-replacement-handoff', 'continuous-full-regression']}
|
||||
original = {str(p): _hash(p) for root in (before, after) for p in root.rglob('*') if p.is_file()}
|
||||
failed_name = '00-invalid/input.prj' if args.fault_batch else None
|
||||
if args.fault_batch:
|
||||
for phase, source in (('before', before), ('after', after)):
|
||||
destination = output/'inputs'/phase
|
||||
shutil.copytree(source, destination)
|
||||
fixture = destination/'00-invalid'; (fixture/'database').mkdir(parents=True)
|
||||
(fixture/'input.prj').write_text('KEY RootBlock "board"\nKEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n', encoding='utf-8')
|
||||
(fixture/'database/icdb.dat').write_bytes(b'deliberately-invalid-acceptance-fixture')
|
||||
before, after = output/'inputs/before', output/'inputs/after'
|
||||
original.update((str(p), _hash(p)) for root in (before, after) for p in root.rglob('*') if p.is_file())
|
||||
report.update(evidence_kind='native-real-copies-and-invalid-database-fixture', expected_failed_items=[failed_name])
|
||||
expected_names = sorted(args.expected_items + ([failed_name] if failed_name else []))
|
||||
process = driver = main_window = None
|
||||
|
||||
def sql(query, params=()):
|
||||
with sqlite3.connect(data/'task-center.db') as db:
|
||||
db.row_factory = sqlite3.Row
|
||||
return [dict(row) for row in db.execute(query, params)]
|
||||
|
||||
@contextmanager
|
||||
def reader():
|
||||
store = TaskStore(data/'task-center.db')
|
||||
try: yield VerificationService(store, data)
|
||||
finally: store.close()
|
||||
|
||||
def launch():
|
||||
nonlocal process, driver, main_window
|
||||
environment = dict(os.environ)
|
||||
for name in ('PYTHONPATH', 'PYTHONHOME', 'DX_TASK_CENTER_HOME', 'QT_QPA_PLATFORM'): environment.pop(name, None)
|
||||
p = subprocess.Popen([str(exe), '--data-root', str(data)], cwd=exe.parent, env=environment,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
process = p
|
||||
d = WindowDriver(p.pid, output); driver = d
|
||||
def ready():
|
||||
try: return d.main()
|
||||
except TimeoutError:
|
||||
require(p.poll() is None, 'Application exited before its main window appeared')
|
||||
return None
|
||||
window = wait(ready, timeout=90); main_window = window
|
||||
d.click(d.find(automation_id='verification_navigation'))
|
||||
return p, d, window
|
||||
|
||||
def close():
|
||||
for window in driver.windows():
|
||||
if win32gui.GetClassName(window.handle)=='#32770':
|
||||
cancel = next((c for c in window.descendants(control_type='Button') if c.element_info.automation_id=='2'), None)
|
||||
if cancel: win32gui.PostMessage(cancel.handle, win32con.BM_CLICK, 0, 0)
|
||||
win32gui.PostMessage(main_window.handle, win32con.WM_CLOSE, 0, 0)
|
||||
require(process.wait(45)==0, 'Application did not exit normally')
|
||||
|
||||
def run_record(run_id):
|
||||
row = sql('SELECT * FROM verification_runs WHERE run_id=?', (run_id,))[0]
|
||||
require(row['state']=='completed', 'Run stopped, blocked or interrupted')
|
||||
return {'run_id': run_id, 'members': json.loads(row['members_json'])}
|
||||
|
||||
def finished(run_id):
|
||||
rows = sql('SELECT state FROM verification_runs WHERE run_id=?', (run_id,))
|
||||
return rows and rows[0]['state'] not in ('queued', 'running', 'stopping')
|
||||
|
||||
def export_report():
|
||||
destination = output/'reports'; destination.mkdir(exist_ok=True)
|
||||
previous = set(destination.glob('verification-*'))
|
||||
driver.click(driver.find('导出报告', kind='Button'))
|
||||
dialog = wait(lambda: next((w for w in driver.windows() if win32gui.GetClassName(w.handle)=='#32770'), None), timeout=15)
|
||||
# The native folder picker has FolderName edit 1152, unlike file picker 1001.
|
||||
field = next(c for c in dialog.descendants(control_type='Edit') if c.element_info.automation_id=='1152')
|
||||
field.iface_value.SetValue(str(destination))
|
||||
require(field.get_value()==str(destination), 'Folder picker path differs from the requested destination')
|
||||
confirm = next(c for c in dialog.descendants(control_type='Button') if c.element_info.automation_id=='1')
|
||||
win32gui.PostMessage(confirm.handle, win32con.BM_CLICK, 0, 0)
|
||||
wait(lambda: not win32gui.IsWindow(dialog.handle), timeout=20)
|
||||
return wait(lambda: next((p for p in destination.glob('verification-*') if p not in previous and (p/'task.json').is_file()), None), timeout=60)
|
||||
|
||||
def selected_cell(name):
|
||||
tables = [w for kind in ('Table', 'DataGrid') for w in main_window.descendants(control_type=kind) if w.is_visible()]
|
||||
table = min(tables, key=lambda w: w.rectangle().left)
|
||||
return next((c for c in table.children() if c.window_text()==name), None)
|
||||
|
||||
def verify_report(scope, run, directory, label, *, selected_only=False):
|
||||
with reader() as service:
|
||||
validation = verify_closed_loop(service, task, scope, run, directory,
|
||||
expected_verdicts={row['item_id']: args.expect_verdict for row in scope['items']}, selected_only=selected_only)
|
||||
(output/(label+'.json')).write_text(json.dumps(validation, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
report['checks'].extend(validation['checks'])
|
||||
require(validation['passed'], label+' failed: '+json.dumps(validation['checks'], ensure_ascii=False))
|
||||
|
||||
try:
|
||||
process, driver, main_window = launch()
|
||||
reduce = driver.find('减少透明度', kind='Button', optional=True)
|
||||
if reduce: driver.click(reduce)
|
||||
driver.click(wait(lambda: driver.find(automation_id='verificationNew')))
|
||||
dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='新建校验'), None))
|
||||
ok = next(b for b in dialog.descendants(control_type='Button') if b.window_text().replace('&', '') in ('OK', '确定'))
|
||||
driver.click(ok)
|
||||
button = wait(lambda: driver.find(automation_id='verificationBatchImport'))
|
||||
wait(lambda: button.is_enabled()); driver.click(button)
|
||||
dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='批量添加工程对'), None))
|
||||
driver.type_text(driver.find(automation_id='verificationBeforeRoot', window=dialog), str(before))
|
||||
driver.type_text(driver.find(automation_id='verificationAfterRoot', window=dialog), str(after))
|
||||
driver.click(driver.find(automation_id='verificationScanPairs', window=dialog))
|
||||
button = driver.find(automation_id='verificationImportPairs', window=dialog)
|
||||
wait(lambda: button.is_enabled()); window_image(dialog.handle).save(output/'01-pairs.png')
|
||||
driver.click(button)
|
||||
wait(lambda: driver.find(f'本批导入完成:已导入 {len(expected_names)} 对,失败 0 对;尚未开始校验。', kind='Text', window=dialog), timeout=120)
|
||||
driver.click(driver.find('关闭', kind='Button', window=dialog))
|
||||
imported = sql('SELECT * FROM verification_items ORDER BY display_name')
|
||||
require([r['display_name'] for r in imported]==expected_names, 'Imported names differ from the predeclared scope')
|
||||
require(not sql('SELECT attempt_id FROM verification_attempts'), 'Import started verification automatically')
|
||||
task = imported[0]['task_id']
|
||||
with reader() as service: scope = freeze_inputs(service, task, expected_count=len(expected_names))
|
||||
(output/'frozen-scope.json').write_text(json.dumps(dict(scope, expected_verdict=args.expect_verdict,
|
||||
expected_failed_items=[failed_name] if failed_name else []), ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
driver.click(driver.find('开始全部', kind='Button'))
|
||||
run_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs')), None))
|
||||
wait(lambda: finished(run_id), timeout=900)
|
||||
run = run_record(run_id)
|
||||
if failed_name:
|
||||
failed_item = next(r['item_id'] for r in imported if r['display_name']==failed_name)
|
||||
with reader() as service:
|
||||
failure = service.read_item(failed_item)
|
||||
require(run['members'][0]['item_id']==failed_item, 'Fault fixture did not run before the healthy projects')
|
||||
require(failure['attempt_state']=='failed' and failure['verdict']=='INCOMPLETE'
|
||||
and failure['attempt']['result']['resource_state']=='released', 'Invalid database did not fail and release resources as expected')
|
||||
report['checks'].append({'check_id': 'C08:failed-native-item-continues', 'state': 'passed',
|
||||
'item_id': failed_item, 'observed_state': failure['attempt_state'], 'observed_verdict': failure['verdict']})
|
||||
report['not_run'][0] = 'C08:packaged-stop-and-uncertain-resources'
|
||||
scope = dict(scope, items=[row for row in scope['items'] if row['item_id']!=failed_item])
|
||||
run = dict(run, members=[row for row in run['members'] if row['item_id']!=failed_item])
|
||||
name = sorted(args.expected_items)[0]
|
||||
driver.click(wait(lambda: selected_cell(name)))
|
||||
label = {'CONSISTENT': '一致', 'WARNING': '需关注', 'DIFFERENT': '异常'}[args.expect_verdict]
|
||||
wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60)
|
||||
window_image(main_window.handle).save(output/'02-result.png')
|
||||
directory = export_report(); verify_report(scope, run, directory, 'first-run', selected_only=bool(failed_name))
|
||||
first_report_hashes = {str(p): _hash(p) for p in directory.iterdir() if p.is_file()}
|
||||
attempts = sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id')
|
||||
report['first_report'] = str(directory)
|
||||
close(); process, driver, main_window = launch()
|
||||
driver.click(wait(lambda: selected_cell(name)))
|
||||
wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60)
|
||||
# Wait for startup recovery and at least one normal workspace polling interval.
|
||||
time.sleep(2)
|
||||
require(sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id')==attempts, 'Restart changed attempts or started another run')
|
||||
verify_report(scope, run, directory, 'restart-readback', selected_only=bool(failed_name))
|
||||
report['checks'].append({'check_id': 'C06:restart', 'state': 'passed'})
|
||||
window_image(main_window.handle).save(output/'03-restart.png')
|
||||
old_ids = {r['run_id'] for r in sql('SELECT run_id FROM verification_runs')}
|
||||
driver.click(driver.find('重校验', kind='Button'))
|
||||
rerun_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs') if r['run_id'] not in old_ids), None))
|
||||
wait(lambda: finished(rerun_id), timeout=900)
|
||||
rerun = run_record(rerun_id)
|
||||
subset = dict(scope, items=[r for r in scope['items'] if r['display_name']==name])
|
||||
member = rerun['members'][0]
|
||||
previous = next(r for r in attempts if r['attempt_id']==next(m['attempt_id'] for m in run['members'] if m['item_id']==member['item_id']))
|
||||
latest = sql('SELECT * FROM verification_attempts WHERE attempt_id=?', (member['attempt_id'],))[0]
|
||||
require(latest['generation']==previous['generation']+1 and latest['input_version']==previous['input_version'], 'Rerun did not create the next attempt on the same input')
|
||||
require(all(_hash(Path(p))==sha for p, sha in first_report_hashes.items()), 'Rerun overwrote the first report')
|
||||
require(sql('SELECT result_json FROM verification_attempts WHERE attempt_id=?', (previous['attempt_id'],))[0]['result_json']==previous['result_json'], 'Rerun changed the old result')
|
||||
second_report = export_report()
|
||||
verify_report(subset, rerun, second_report, 'rerun', selected_only=True)
|
||||
report['checks'].append({'check_id': 'C07:rerun', 'state': 'passed'})
|
||||
report['rerun_report'] = str(second_report)
|
||||
require(all(_hash(Path(p))==sha for p, sha in original.items()), 'Input files changed during the loop')
|
||||
require(sql('SELECT COUNT(*) total FROM execution_operations')[0]['total']==0, 'Replacement operations occurred')
|
||||
close(); report['passed'] = True
|
||||
except Exception as error:
|
||||
report.update(passed=False, reason=str(error), error=type(error).__name__)
|
||||
if driver:
|
||||
for n, window in enumerate(driver.windows()):
|
||||
try: window_image(window.handle).save(output/f'failure-{n}.png')
|
||||
except Exception: pass
|
||||
if process and process.poll() is None and main_window:
|
||||
try: close()
|
||||
except Exception: report['process_left_running'] = process.pid
|
||||
elif process and process.poll() is None:
|
||||
report['process_left_running'] = process.pid
|
||||
finally:
|
||||
report['input_sha256'] = original
|
||||
(output/'acceptance.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
return 0 if report.get('passed') else 1
|
||||
|
||||
|
||||
if __name__=='__main__': raise SystemExit(main())
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Independent acceptance checks against a scope recorded BEFORE execution."""
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from dx_productivity.trial_task_center.native_netlist import _hash
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def records(service, task_id):
|
||||
result, cursor = [], None
|
||||
while True:
|
||||
page = service.read_task(task_id, cursor=cursor)
|
||||
result.extend(service.read_item(row['item_id']) for row in page['items'])
|
||||
cursor = page['cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
|
||||
|
||||
def freeze_inputs(service, task_id, *, expected_count):
|
||||
rows = records(service, task_id)
|
||||
require(type(expected_count) is int and expected_count>0 and len(rows)==expected_count,
|
||||
'验收必须事先指定非空、准确的工程数量。')
|
||||
sources, items = {}, []
|
||||
for row in rows:
|
||||
require(row['input_state']=='ready', '开始前输入尚未就绪。')
|
||||
payload = row['payload']
|
||||
for name in ('before_ref', 'after_ref'):
|
||||
capture = service.artifacts.read(payload[name], 'snapshot')
|
||||
for path, sha in capture['source_hashes'].items():
|
||||
require(path not in sources or sources[path]==sha, '同一源文件出现不同摘要。')
|
||||
require(_hash(Path(path))==sha, '源文件已偏离导入时的版本。')
|
||||
sources[path] = sha
|
||||
items.append({'item_id': row['item_id'], 'input_version': row['input_version'],
|
||||
'input_digest': payload['payload_digest'], 'before_ref': payload['before_ref'],
|
||||
'after_ref': payload['after_ref'], 'policy': payload['check_plan'], 'display_name': row['display_name']})
|
||||
operations = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]
|
||||
return {'standard': 'verification-closed-loop-v1', 'task_id': task_id, 'items': items,
|
||||
'source_hashes': sources, 'replacement_operations_before': operations}
|
||||
|
||||
|
||||
class _ReportHTML(HTMLParser):
|
||||
def __init__(self, text):
|
||||
super().__init__(); self.mains = []; self.diagrams = 0
|
||||
self.feed(text)
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag=='main': self.mains.append(dict(attrs))
|
||||
if tag=='svg': self.diagrams += 1
|
||||
|
||||
|
||||
def verify_closed_loop(service, task_id, scope, run, report, *, expected_verdicts, selected_only=False):
|
||||
checks = []
|
||||
def check(name, action):
|
||||
try:
|
||||
action()
|
||||
checks.append({'check_id': name, 'state': 'passed'})
|
||||
except (ValueError, OSError, KeyError, TypeError) as error:
|
||||
checks.append({'check_id': name, 'state': 'failed', 'reason': str(error)})
|
||||
|
||||
frozen = {r['item_id']: r for r in scope['items']}
|
||||
members = {r['item_id']: r for r in run['members']}
|
||||
report = Path(report)
|
||||
current = {r['item_id']: r for r in records(service, task_id)}
|
||||
if selected_only:
|
||||
current = {key: row for key, row in current.items() if key in frozen}
|
||||
def membership():
|
||||
require(scope['task_id']==task_id and bool(frozen), '冻结任务为空或身份错误。')
|
||||
require(len(frozen)==len(scope['items']) and len(members)==len(run['members']), '工程或运行成员重复。')
|
||||
require(set(frozen)==set(members)==set(current)==set(expected_verdicts), '冻结范围、运行成员、当前工程或预期清单不一致。')
|
||||
durable = service.repo.connection.execute('SELECT state,members_json FROM verification_runs WHERE run_id=?', (run['run_id'],)).fetchone()
|
||||
require(durable is not None and durable['state']=='completed', '运行未完成。')
|
||||
persisted = json.loads(durable['members_json'])
|
||||
require(all(row in persisted for row in run['members']) if selected_only else persisted==run['members'],
|
||||
'运行成员与持久记录不同。')
|
||||
require(all(v in ('CONSISTENT', 'WARNING', 'DIFFERENT') for v in expected_verdicts.values()),
|
||||
'正常完整闭环必须事先指定可核验的工程结论。')
|
||||
check('C01-C02:scope', membership)
|
||||
|
||||
saved = {}
|
||||
def report_scope():
|
||||
value = json.loads((report/'task.json').read_text(encoding='utf-8'))
|
||||
require(value['task_id']==task_id, '报告任务不符。')
|
||||
identities = {r['item_id'] for r in value['items']}
|
||||
require(len(identities)==len(value['items']) and (set(frozen)<=identities if selected_only else identities==set(frozen)), '报告工程范围不符。')
|
||||
require((report/'index.html').is_file(), '缺少报告首页。')
|
||||
saved.update((r['item_id'], r) for r in value['items'])
|
||||
check('C04:report_scope', report_scope)
|
||||
|
||||
for item, original in frozen.items():
|
||||
def item_result(item=item, original=original):
|
||||
row, member = current[item], members[item]
|
||||
require(row['input_version']==member['input_version']==original['input_version'], '输入版本不符。')
|
||||
require(row['payload']['payload_digest']==original['input_digest'], '输入摘要变化。')
|
||||
for key in ('before_ref', 'after_ref'):
|
||||
require(row['payload'][key]==original[key], '冻结副本引用变化。')
|
||||
attempt = row['attempt']
|
||||
require(attempt is not None and attempt['state']=='succeeded', '尝试未成功完成,停止/失败不能算闭环。')
|
||||
require(attempt['run_id']==run['run_id'] and row['attempt_id']==member['attempt_id'], '尝试不属于本次运行。')
|
||||
service.validate_attempt(attempt)
|
||||
result = attempt['result']; evidence = result['evidence']; binding = evidence['binding']
|
||||
require(result['verdict']==expected_verdicts[item], '工程结论与预期不符。')
|
||||
for key, expected in (('item_id', item), ('attempt_id', attempt['attempt_id']),
|
||||
('generation', attempt['generation']), ('input_version', original['input_version']),
|
||||
('input_digest', original['input_digest']), ('policy', original['policy'])):
|
||||
require(result[key]==expected, '结果身份不符:'+key)
|
||||
for key in ('attempt_id', 'input_version', 'input_digest'):
|
||||
require(binding[key]==result[key], '图表证据绑定不符:'+key)
|
||||
require(result['resource_state']=='released', '执行资源退出未确认。')
|
||||
released = json.loads((service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').read_text(encoding='utf-8'))
|
||||
require(released['resource_state']=='released' and released['owner_id']==attempt['owner_id'], '资源退出证明不属于本次尝试。')
|
||||
if result['verdict'] in ('CONSISTENT', 'WARNING'):
|
||||
states = {c['check_id']: c['state'] for c in result['checks']}
|
||||
require(all(states.get(key) in ('equal', 'expected_change', 'warning')
|
||||
for key in original['policy']['required_checks']), '必查项缺失或未知。')
|
||||
else:
|
||||
require(any(c['state']=='error' for c in result['checks']), '异常结论缺少错误事实。')
|
||||
published = saved[item]
|
||||
require(published['input_version']==original['input_version'] and published['attempt_id']==attempt['attempt_id'],
|
||||
'报告引用旧输入或旧尝试。')
|
||||
require(published['attempt']['result']==result, 'JSON报告事实与当前结果不同。')
|
||||
html = _ReportHTML((report/f'{item}-v{original["input_version"]}.html').read_text(encoding='utf-8'))
|
||||
require(len(html.mains)==1, 'HTML报告缺少唯一绑定。')
|
||||
for key, value in (('item-id', item), ('attempt-id', attempt['attempt_id']), ('generation', attempt['generation']),
|
||||
('input-version', original['input_version']), ('input-digest', original['input_digest']),
|
||||
('evidence-digest', evidence['digest'])):
|
||||
require(html.mains[0].get('data-'+key)==str(value), 'HTML报告绑定不符:'+key)
|
||||
require(not evidence['pins'] or html.diagrams>0, '有逐脚事实却没有报告接线图。')
|
||||
check('C02-C04:'+item, item_result)
|
||||
|
||||
def unchanged():
|
||||
require(bool(scope['source_hashes']), '没有冻结源文件摘要。')
|
||||
require(all(_hash(Path(path))==sha for path, sha in scope['source_hashes'].items()), '源文件被改写。')
|
||||
count = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]
|
||||
# Early v1 standalone fixtures recorded an implicit zero baseline.
|
||||
require(count==scope.get('replacement_operations_before', 0), '校验期间出现了新的换料操作或历史记录变化。')
|
||||
check('C05:read_only', unchanged)
|
||||
return {'standard': scope['standard'], 'task_id': task_id, 'run_id': run['run_id'],
|
||||
'validation_scope': 'single-run:C01-C05', 'not_run': ['C06', 'C07', 'C08'],
|
||||
'passed': all(c['state']=='passed' for c in checks), 'checks': checks}
|
||||
@@ -12,6 +12,7 @@ from dx_productivity.trial_task_center.store import TaskStore
|
||||
from dx_productivity.trial_task_center.verification.service import VerificationService
|
||||
from dx_productivity.trial_task_center import native_netlist,netlist_verification
|
||||
from dx_productivity.trial_task_center.batch_resources import write_record
|
||||
from verification_closed_loop import freeze_inputs,verify_closed_loop
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
@@ -20,6 +21,8 @@ def main(argv=None):
|
||||
group.add_argument('--fixture-root',type=Path)
|
||||
group.add_argument('--native-before',type=Path)
|
||||
parser.add_argument('--native-after',type=Path);parser.add_argument('--output',type=Path,required=True)
|
||||
parser.add_argument('--expect-verdict',choices=('CONSISTENT','WARNING','DIFFERENT'),default='CONSISTENT',
|
||||
help='Expected conclusion fixed before running; defaults to CONSISTENT')
|
||||
args=parser.parse_args(argv)
|
||||
if args.native_before and not args.native_after:parser.error('--native-after is required')
|
||||
root=args.output.resolve()
|
||||
@@ -48,6 +51,9 @@ def main(argv=None):
|
||||
task=service.create_task('独立校验验收',request_id='acceptance-create')['task_id']
|
||||
accepted=service.import_pair(task,before,after,request_id='acceptance-import',stop_event=Event())
|
||||
if not accepted['accepted']:raise ValueError('No input accepted')
|
||||
frozen=freeze_inputs(service,task,expected_count=1)
|
||||
expectations={row['item_id']:args.expect_verdict for row in frozen['items']}
|
||||
write_record(root/'frozen-scope.json',dict(frozen,expected_verdicts=expectations,evidence_kind=evidence_kind))
|
||||
run=service.start(task,request_id='acceptance-run')
|
||||
deadline=time.monotonic()+900
|
||||
while service.busy:
|
||||
@@ -55,13 +61,15 @@ def main(argv=None):
|
||||
time.sleep(.05)
|
||||
current=service.read_task(task);report=service.export_task(task,root/'reports',request_id='acceptance-report')
|
||||
unchanged=all(native_netlist._hash(Path(p))==sha for p,sha in initial.items())
|
||||
closed_loop=verify_closed_loop(service,task,frozen,run,report,expected_verdicts=expectations)
|
||||
write_record(root/'closed-loop.json',closed_loop)
|
||||
summary={'evidence_kind':evidence_kind,'task_id':task,'run':run,'summary':current['summary'],
|
||||
'input_sha256':initial,'inputs_unchanged':unchanged,'report':str(report),
|
||||
'policy':'verification-v1','replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]}
|
||||
'policy':'verification-v1','closed_loop':closed_loop,
|
||||
'replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]}
|
||||
write_record(root/'acceptance.json',summary);print(json.dumps(summary,ensure_ascii=False))
|
||||
if not unchanged:return 4
|
||||
if current['summary']['different']:return 2
|
||||
if current['summary']['unavailable'] or current['summary']['waiting'] or current['summary']['running']:return 3
|
||||
if not closed_loop['passed']:return 3
|
||||
return 0
|
||||
except Exception as error:
|
||||
write_record(root/'failure.json',{'error':type(error).__name__,'reason':str(error)})
|
||||
|
||||
Reference in new issue
Block a user