diff --git a/docs/superpowers/plans/2026-11-17-verification-development.md b/docs/superpowers/plans/2026-11-17-verification-development.md index bb8a499..06261e6 100644 --- a/docs/superpowers/plans/2026-11-17-verification-development.md +++ b/docs/superpowers/plans/2026-11-17-verification-development.md @@ -138,6 +138,21 @@ input_record = { ## 8. 交接状态 +### N12 后续闭环标准(2026-11-18) + +所有后续任务按[闭环标准 v1](../../testing/verification-closed-loop-standard.md)留证;不再用“导入成功”“文件已生成”或历史分组通过数量单独宣布闭环完成。批量添加见[验收记录](../../testing/2026-11-18-verification-batch-import.md)。 + +- [x] C01~C05:开始前冻结名单/输入/策略,核对实际运行成员、预期结论、同源图表与报告、原文和退出证明;源文件不变、新增换料操作为0。 +- [x] C06:实际包正常关闭后重启,证据可回读,尝试数不增加。 +- [x] C07:实际包单项重校验,新attempt及generation递增,旧结果/报告不被覆盖。 +- [x] C08受控回归:反脚、命名警告、必查未知、停止/资源未知、版本竞争和单项失败后继续的服务/Qt反例。 +- [ ] C08实际包补充:原生导出失败项先运行,后续正常项仍完成;本轮正在执行并记录独立现场。 +- [ ] C08实际包停止与资源未知:需要各自独立的受控现场,正常路径不替代。 +- [ ] 真实已保存换料回执触发的自动交接实际包闭环;合同/幂等测试已通过,不能冒用独立输入证明。 +- [ ] 连续默认全量稳定全绿;前轮既有时序波动仍交由原修复任务处理,不改旧断言。 + +单次只读验收入口为 `tools/verify_independent_verification.py --expect-verdict ...`,明确只覆盖 C01~C05;实际包入口为 `tools/accept_verification_closed_loop.py`,覆盖独立输入 C01~C07,并可用 `--fault-batch` 增加原生失败隔离反例。未测项在JSON里保留not_run。测试失败或标准不符均非零退出;已有换料历史采用执行前基线,不能要求历史总数为0。 + 用户授权的 N01~N12 已顺序实现、分步提交并完成本地开发包验收。新增组 81 项通过,实际包已验证独立导入、原生导出、查看证据、重校验和正常关闭。既有连续全量回归的两处时序波动仍保留,未合并、推送或发布。本分支没有复制另一修复任务的未提交修改。 实施进度及验证结果见 docs/testing/2026-11-17-independent-verification-acceptance.md。后续开发以本入口和[N01~N12清单](2026-11-17-independent-verification-new-features.md)共同为准,11月6日的V/U旧计划保留为历史参考,不能再并行照旧重建后端或来源界面。 diff --git a/docs/testing/verification-closed-loop-standard.md b/docs/testing/verification-closed-loop-standard.md new file mode 100644 index 0000000..94fb9b6 --- /dev/null +++ b/docs/testing/verification-closed-loop-standard.md @@ -0,0 +1,35 @@ +# 工程校验闭环标准 v1 + +本标准用于 N12 及后续功能。写完代码、测试通过、工程检查一致、业务闭环通过、允许发布,是不同结论。每次验收必须声明正在证明哪一个结论。 + +## 单次功能闭环的通过条件 + +| 编号 | 必须证明 | 不满足时 | +| --- | --- | --- | +| C01 输入冻结 | 开始前写下预期工程名单、数量、前后摘要、输入版本和检查策略;配对预览与实际导入相符 | 失败;不能从运行后的结果反推预期 | +| C02 完整执行 | 用户显式开始;运行成员与冻结范围完全相同,每项都有属于本次运行的终态尝试 | 有排队、运行、停止、中断、漏项或重复项,均不能算完整闭环 | +| C03 判断正确 | 每项结论符合事先指定的预期;执行成功与 CONSISTENT/WARNING/DIFFERENT 分开;必查未知不得通过 | 误报、漏报、未知充当一致都失败 | +| C04 事实同源 | 页面、逐脚表、JSON和HTML报告绑定同一 item、输入版本、attempt、generation、输入摘要和证据摘要;原文摘要可复核 | 任何串版本、旧结果、缺证据或报告不符都失败 | +| C05 只读安全 | 校验前后全部已冻结源文件摘要不变,新增换料操作数为0,执行资源退出有证明;已有换料历史以开始前基线计数核对 | 不变性或退出证明缺失,失败并保留现场 | +| C06 重启回读 | 正常退出后重启,结果及证据仍可查看,尝试数不增加、不自动重跑 | 失败 | +| C07 重校验 | 同一输入重新执行时 attempt 更新、generation 递增;旧结果保留,新旧报告均可追溯 | 失败 | +| C08 异常隔离 | 反脚/断线等业务异常不挡后续工程;停止后未派发项仍待校验;资源/存储未知暂停正确范围;仍可新建独立任务 | 相应异常用例失败;不能以“正常样本跑过”替代 | + +只有本次要求的检查全部通过,才写“该功能闭环通过”。验证程序对失败必须非零退出;结构性检查失败不能被 WARNING、正常进程退出或成功生成文件覆盖。某项故障测试验证了预期失败,不等于故障工程已经检查通过。 + +## 固定业务反例 + +物理脚1/2互换、断线、短接、位号或物理脚号变化必须为 DIFFERENT;loop→outloop 且连接不变必须为 WARNING;必查字段缺失必须为 INCOMPLETE。图中固定物理脚号位置,异常端点可与逐脚表和原文对应。自建格式夹具用于反例和可复现测试,必须明确标注,不能当成真实CAD导出证据。 + +## 两条业务路径分别验收 + +1. 独立输入:实际开发包中批量添加 → 开始全部 → 原生导出 → 查看图表 → 导出报告 → 正常关闭 → 重启查看 → 重校验 → 再导出。 +2. 成功结果交接:真正已落盘的成功换料回执 → 自动登记校验 → 同一套执行、复核与报告。重复接收不重复建项,未保存项不得作为成功项。 + +两条路径不能互相代替。受控服务测试可证明交接合同、幂等和非阻塞;没有取得实际成功回执触发的包内证据时,该路径记“实际闭环未测”。本任务不为补验收重新执行客户换料,也不修改其他任务的旧执行器。 + +## 留证与发布 + +验收记录至少包含:代码提交、EXE SHA256、证据类型、预期清单、输入和报告摘要、任务/运行/尝试身份、每项标准的 passed/failed/not_run、失败原因、截图和实际命令。前一次运行、仅有截图、历史分组测试覆盖均不能替代本次闭环。 + +发布还要求相关回归通过、连续默认全量无未解决失败、实际包身份与源码相符、声明的两条实际路径通过。现有时序波动仍是发布保留项,不在本功能中改断言掩盖。引脚级Value和确切库内脚到焊盘映射仍未覆盖;不能写成“封装全部正确”。 diff --git a/src/dx_productivity/trial_task_center/verification/report.py b/src/dx_productivity/trial_task_center/verification/report.py index 3a530a0..cb89ec1 100644 --- a/src/dx_productivity/trial_task_center/verification/report.py +++ b/src/dx_productivity/trial_task_center/verification/report.py @@ -76,7 +76,11 @@ def item_html(result): def render_item_report(result_ref,destination): destination=Path(destination) if destination.exists():fail('报告文件已存在,请换一个名称。') - html='工程校验报告
'+item_html(result_ref)+'
' + fields={'item-id':result_ref.get('item_id'),'attempt-id':result_ref.get('attempt_id'), + 'input-version':result_ref.get('input_version'),'generation':result_ref.get('generation'), + 'input-digest':result_ref.get('input_digest'),'evidence-digest':(result_ref.get('evidence') or {}).get('digest')} + binding=' '.join(f'data-{key}="{text(value)}"' for key,value in fields.items()) + html='工程校验报告
'+item_html(result_ref)+'
' destination.parent.mkdir(parents=True,exist_ok=True) temporary=destination.with_name(destination.name+'.'+uuid4().hex+'.tmp') try: diff --git a/tests/v1_1/test_verification_closed_loop.py b/tests/v1_1/test_verification_closed_loop.py new file mode 100644 index 0000000..f3f911a --- /dev/null +++ b/tests/v1_1/test_verification_closed_loop.py @@ -0,0 +1,114 @@ +import copy +import json +from pathlib import Path +import pytest + +from test_independent_verification_intake import service +from test_independent_verification_runner import imported +from test_native_cli_netlist import graph +from test_verification_control import wait_idle + + +def complete(service, tmp_path, monkeypatch): + from tools.verification_closed_loop import freeze_inputs + from dx_productivity.trial_task_center import netlist_verification + task, item = imported(service, tmp_path) + scope = freeze_inputs(service, task, expected_count=1) + def native(capture, output, *, stop_event): + output.mkdir(parents=True, exist_ok=True) + return {'status': 'success', 'complete': True, 'capture': capture, 'graph': graph(output), + 'outputs': {'effective': str(output/'effective.txt'), 'keyin': str(output/'design.kyn')}} + monkeypatch.setattr(netlist_verification, '_export_saved_capture', native) + run = service.start(task, request_id='loop-start'); wait_idle(service) + report = service.export_task(task, tmp_path/'reports', request_id='loop-report') + return task, item, scope, run, report + + +def test_complete_scope_report_and_inputs_pass_same_attempt_checks(service, tmp_path, monkeypatch): + from tools.verification_closed_loop import verify_closed_loop + task, item, scope, run, report = complete(service, tmp_path, monkeypatch) + result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'}) + assert result['passed'] and all(c['state']=='passed' for c in result['checks']) + + +def test_empty_task_cannot_be_frozen_as_a_closed_loop(service): + from tools.verification_closed_loop import freeze_inputs + task = service.create_task('空', request_id='empty')['task_id'] + with pytest.raises(ValueError): freeze_inputs(service, task, expected_count=0) + + +@pytest.mark.parametrize('fault', ['run_member', 'report_attempt', 'html_binding', 'source', 'expected_verdict', 'resource_proof', 'evidence_binding']) +def test_closed_loop_rejects_wrong_members_stale_reports_and_changed_sources(service, tmp_path, monkeypatch, fault): + from tools.verification_closed_loop import verify_closed_loop + task, item, scope, run, report = complete(service, tmp_path, monkeypatch) + expected = {item: 'CONSISTENT'} + if fault=='run_member': + run = copy.deepcopy(run); run['members'] = [] + elif fault=='report_attempt': + path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8')) + value['items'][0]['attempt_id'] = 'old-attempt' + path.write_text(json.dumps(value), encoding='utf-8') + elif fault=='html_binding': + path = report/f'{item}-v1.html'; path.write_text('CONSISTENT', encoding='utf-8') + elif fault=='source': + capture = service.artifacts.read(scope['items'][0]['before_ref'], 'snapshot') + Path(capture['project']).write_text('changed', encoding='utf-8') + elif fault=='expected_verdict': + expected[item] = 'DIFFERENT' + elif fault=='resource_proof': + attempt = service.repo.latest_attempt(item, 1) + (service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').write_text( + json.dumps({'owner_id': 'another-owner', 'resource_state': 'released'}), encoding='utf-8') + else: + path = report/'task.json'; value = json.loads(path.read_text(encoding='utf-8')) + value['items'][0]['attempt']['result']['evidence']['binding']['input_digest'] = 'f'*64 + path.write_text(json.dumps(value), encoding='utf-8') + result = verify_closed_loop(service, task, scope, run, report, expected_verdicts=expected) + assert not result['passed'] and any(c['state']=='failed' for c in result['checks']) + + +def test_stopped_attempt_is_not_a_completed_closed_loop(service, tmp_path): + from tools.verification_closed_loop import freeze_inputs, verify_closed_loop + task, item = imported(service, tmp_path) + scope = freeze_inputs(service, task, expected_count=1) + attempt = service.repo.create_attempt(item, 1, request_id='cancelled') + service.repo.finish(attempt['attempt_id'], generation=attempt['generation'], result={ + 'attempt_state': 'cancelled', 'resource_state': 'released', 'verdict': 'INCOMPLETE', 'reason': 'stop'}) + run = {'run_id': None, 'members': [{'item_id': item, 'input_version': 1, 'attempt_id': attempt['attempt_id']}]} + report = service.export_task(task, tmp_path/'reports', request_id='cancelled-report') + result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'INCOMPLETE'}) + assert not result['passed'] + + +def test_selected_rerun_is_checked_against_its_scope_inside_a_whole_task_report(service, tmp_path, monkeypatch): + from tools.verification_closed_loop import freeze_inputs, verify_closed_loop + task, item, _, _, _ = complete(service, tmp_path, monkeypatch) + payload = {key: value for key, value in service.repo.revision(item, 1)['payload'].items() + if key not in ('payload_digest', 'input_state')} + payload.update(item_key='second', display_name='second') + service.publish_inputs(task, (payload,), submission_id='second') + scope = freeze_inputs(service, task, expected_count=2) + whole = service.start(task, selection=tuple((row['item_id'], 1) for row in scope['items']), request_id='whole-task'); wait_idle(service) + subset = dict(scope, items=[row for row in scope['items'] if row['item_id']==item]) + selected_run = dict(whole, members=[row for row in whole['members'] if row['item_id']==item]) + whole_report = service.export_task(task, tmp_path/'whole-report', request_id='whole-report') + assert verify_closed_loop(service, task, subset, selected_run, whole_report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True)['passed'] + run = service.rerun(item, 1, request_id='selected'); wait_idle(service) + scope['items'] = [row for row in scope['items'] if row['item_id']==item] + report = service.export_task(task, tmp_path/'selected-report', request_id='selected-report') + result = verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'}, selected_only=True) + assert result['passed'] + assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed'] + + +def test_existing_replacement_history_is_allowed_but_new_replacement_is_rejected(service, tmp_path, monkeypatch): + from tools.verification_closed_loop import verify_closed_loop + def operation(key): + with service.repo.store.transaction(): + service.repo.connection.execute('''INSERT INTO execution_operations + (task_id,round_id,operation_key,command_json,state) VALUES ('prior','round',?,'{}','verified')''', (key,)) + operation('before-verification') + task, item, scope, run, report = complete(service, tmp_path, monkeypatch) + assert verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed'] + operation('during-verification') + assert not verify_closed_loop(service, task, scope, run, report, expected_verdicts={item: 'CONSISTENT'})['passed'] diff --git a/tools/accept_verification_closed_loop.py b/tools/accept_verification_closed_loop.py new file mode 100644 index 0000000..bac87d3 --- /dev/null +++ b/tools/accept_verification_closed_loop.py @@ -0,0 +1,232 @@ +"""Actual Windows EXE: batch import, native run, report, restart, rerun, report. + +Use saved test copies. Expectations must be supplied before launch. This tool +never creates replacement receipts or changes task state directly in SQLite. +""" +import argparse +from contextlib import contextmanager +import json +import os +import shutil +from pathlib import Path +import sqlite3 +import subprocess +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'src')) +from dx_productivity.trial_task_center.store import TaskStore +from dx_productivity.trial_task_center.verification.service import VerificationService +from dx_productivity.trial_task_center.native_netlist import _hash +from verification_closed_loop import freeze_inputs, verify_closed_loop, require + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('exe', 'before-root', 'after-root', 'output'): + parser.add_argument('--'+name, type=Path, required=True) + parser.add_argument('--expected-items', nargs='+', required=True, help='Exact relative paths, e.g. A/input.prj B/input.prj') + parser.add_argument('--expect-verdict', choices=('CONSISTENT', 'WARNING', 'DIFFERENT'), required=True) + parser.add_argument('--fault-batch', action='store_true', help='Use private copies and prepend one deliberately invalid database fixture') + args = parser.parse_args(argv) + output = args.output.resolve(); exe = args.exe.resolve() + before, after = args.before_root.resolve(), args.after_root.resolve() + if output.exists(): parser.error('output must be a new directory') + if output.is_relative_to(before) or output.is_relative_to(after): parser.error('output must be outside input roots') + if len(set(args.expected_items))!=len(args.expected_items): parser.error('expected-items must be unique') + output.mkdir(parents=True) + # Import UI automation only for the EXE path; service acceptance remains portable. + import win32con + import win32gui + from packaged_window_driver import WindowDriver, window_image + from accept_packaged_ui import wait + data = output/'data' + report = {'standard': 'verification-closed-loop-v1', 'validation_scope': 'independent-native:C01-C07', + 'evidence_kind': 'native-packaged-ui', 'executable': str(exe), 'executable_sha256': _hash(exe), + 'expected_items': args.expected_items, 'expected_verdict': args.expect_verdict, 'checks': [], + 'not_run': ['C08:packaged-fault-cases', 'actual-successful-replacement-handoff', 'continuous-full-regression']} + original = {str(p): _hash(p) for root in (before, after) for p in root.rglob('*') if p.is_file()} + failed_name = '00-invalid/input.prj' if args.fault_batch else None + if args.fault_batch: + for phase, source in (('before', before), ('after', after)): + destination = output/'inputs'/phase + shutil.copytree(source, destination) + fixture = destination/'00-invalid'; (fixture/'database').mkdir(parents=True) + (fixture/'input.prj').write_text('KEY RootBlock "board"\nKEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n', encoding='utf-8') + (fixture/'database/icdb.dat').write_bytes(b'deliberately-invalid-acceptance-fixture') + before, after = output/'inputs/before', output/'inputs/after' + original.update((str(p), _hash(p)) for root in (before, after) for p in root.rglob('*') if p.is_file()) + report.update(evidence_kind='native-real-copies-and-invalid-database-fixture', expected_failed_items=[failed_name]) + expected_names = sorted(args.expected_items + ([failed_name] if failed_name else [])) + process = driver = main_window = None + + def sql(query, params=()): + with sqlite3.connect(data/'task-center.db') as db: + db.row_factory = sqlite3.Row + return [dict(row) for row in db.execute(query, params)] + + @contextmanager + def reader(): + store = TaskStore(data/'task-center.db') + try: yield VerificationService(store, data) + finally: store.close() + + def launch(): + nonlocal process, driver, main_window + environment = dict(os.environ) + for name in ('PYTHONPATH', 'PYTHONHOME', 'DX_TASK_CENTER_HOME', 'QT_QPA_PLATFORM'): environment.pop(name, None) + p = subprocess.Popen([str(exe), '--data-root', str(data)], cwd=exe.parent, env=environment, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=subprocess.CREATE_NO_WINDOW) + process = p + d = WindowDriver(p.pid, output); driver = d + def ready(): + try: return d.main() + except TimeoutError: + require(p.poll() is None, 'Application exited before its main window appeared') + return None + window = wait(ready, timeout=90); main_window = window + d.click(d.find(automation_id='verification_navigation')) + return p, d, window + + def close(): + for window in driver.windows(): + if win32gui.GetClassName(window.handle)=='#32770': + cancel = next((c for c in window.descendants(control_type='Button') if c.element_info.automation_id=='2'), None) + if cancel: win32gui.PostMessage(cancel.handle, win32con.BM_CLICK, 0, 0) + win32gui.PostMessage(main_window.handle, win32con.WM_CLOSE, 0, 0) + require(process.wait(45)==0, 'Application did not exit normally') + + def run_record(run_id): + row = sql('SELECT * FROM verification_runs WHERE run_id=?', (run_id,))[0] + require(row['state']=='completed', 'Run stopped, blocked or interrupted') + return {'run_id': run_id, 'members': json.loads(row['members_json'])} + + def finished(run_id): + rows = sql('SELECT state FROM verification_runs WHERE run_id=?', (run_id,)) + return rows and rows[0]['state'] not in ('queued', 'running', 'stopping') + + def export_report(): + destination = output/'reports'; destination.mkdir(exist_ok=True) + previous = set(destination.glob('verification-*')) + driver.click(driver.find('导出报告', kind='Button')) + dialog = wait(lambda: next((w for w in driver.windows() if win32gui.GetClassName(w.handle)=='#32770'), None), timeout=15) + # The native folder picker has FolderName edit 1152, unlike file picker 1001. + field = next(c for c in dialog.descendants(control_type='Edit') if c.element_info.automation_id=='1152') + field.iface_value.SetValue(str(destination)) + require(field.get_value()==str(destination), 'Folder picker path differs from the requested destination') + confirm = next(c for c in dialog.descendants(control_type='Button') if c.element_info.automation_id=='1') + win32gui.PostMessage(confirm.handle, win32con.BM_CLICK, 0, 0) + wait(lambda: not win32gui.IsWindow(dialog.handle), timeout=20) + return wait(lambda: next((p for p in destination.glob('verification-*') if p not in previous and (p/'task.json').is_file()), None), timeout=60) + + def selected_cell(name): + tables = [w for kind in ('Table', 'DataGrid') for w in main_window.descendants(control_type=kind) if w.is_visible()] + table = min(tables, key=lambda w: w.rectangle().left) + return next((c for c in table.children() if c.window_text()==name), None) + + def verify_report(scope, run, directory, label, *, selected_only=False): + with reader() as service: + validation = verify_closed_loop(service, task, scope, run, directory, + expected_verdicts={row['item_id']: args.expect_verdict for row in scope['items']}, selected_only=selected_only) + (output/(label+'.json')).write_text(json.dumps(validation, ensure_ascii=False, indent=2), encoding='utf-8') + report['checks'].extend(validation['checks']) + require(validation['passed'], label+' failed: '+json.dumps(validation['checks'], ensure_ascii=False)) + + try: + process, driver, main_window = launch() + reduce = driver.find('减少透明度', kind='Button', optional=True) + if reduce: driver.click(reduce) + driver.click(wait(lambda: driver.find(automation_id='verificationNew'))) + dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='新建校验'), None)) + ok = next(b for b in dialog.descendants(control_type='Button') if b.window_text().replace('&', '') in ('OK', '确定')) + driver.click(ok) + button = wait(lambda: driver.find(automation_id='verificationBatchImport')) + wait(lambda: button.is_enabled()); driver.click(button) + dialog = wait(lambda: next((w for w in driver.windows() if w.window_text()=='批量添加工程对'), None)) + driver.type_text(driver.find(automation_id='verificationBeforeRoot', window=dialog), str(before)) + driver.type_text(driver.find(automation_id='verificationAfterRoot', window=dialog), str(after)) + driver.click(driver.find(automation_id='verificationScanPairs', window=dialog)) + button = driver.find(automation_id='verificationImportPairs', window=dialog) + wait(lambda: button.is_enabled()); window_image(dialog.handle).save(output/'01-pairs.png') + driver.click(button) + wait(lambda: driver.find(f'本批导入完成:已导入 {len(expected_names)} 对,失败 0 对;尚未开始校验。', kind='Text', window=dialog), timeout=120) + driver.click(driver.find('关闭', kind='Button', window=dialog)) + imported = sql('SELECT * FROM verification_items ORDER BY display_name') + require([r['display_name'] for r in imported]==expected_names, 'Imported names differ from the predeclared scope') + require(not sql('SELECT attempt_id FROM verification_attempts'), 'Import started verification automatically') + task = imported[0]['task_id'] + with reader() as service: scope = freeze_inputs(service, task, expected_count=len(expected_names)) + (output/'frozen-scope.json').write_text(json.dumps(dict(scope, expected_verdict=args.expect_verdict, + expected_failed_items=[failed_name] if failed_name else []), ensure_ascii=False, indent=2), encoding='utf-8') + driver.click(driver.find('开始全部', kind='Button')) + run_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs')), None)) + wait(lambda: finished(run_id), timeout=900) + run = run_record(run_id) + if failed_name: + failed_item = next(r['item_id'] for r in imported if r['display_name']==failed_name) + with reader() as service: + failure = service.read_item(failed_item) + require(run['members'][0]['item_id']==failed_item, 'Fault fixture did not run before the healthy projects') + require(failure['attempt_state']=='failed' and failure['verdict']=='INCOMPLETE' + and failure['attempt']['result']['resource_state']=='released', 'Invalid database did not fail and release resources as expected') + report['checks'].append({'check_id': 'C08:failed-native-item-continues', 'state': 'passed', + 'item_id': failed_item, 'observed_state': failure['attempt_state'], 'observed_verdict': failure['verdict']}) + report['not_run'][0] = 'C08:packaged-stop-and-uncertain-resources' + scope = dict(scope, items=[row for row in scope['items'] if row['item_id']!=failed_item]) + run = dict(run, members=[row for row in run['members'] if row['item_id']!=failed_item]) + name = sorted(args.expected_items)[0] + driver.click(wait(lambda: selected_cell(name))) + label = {'CONSISTENT': '一致', 'WARNING': '需关注', 'DIFFERENT': '异常'}[args.expect_verdict] + wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60) + window_image(main_window.handle).save(output/'02-result.png') + directory = export_report(); verify_report(scope, run, directory, 'first-run', selected_only=bool(failed_name)) + first_report_hashes = {str(p): _hash(p) for p in directory.iterdir() if p.is_file()} + attempts = sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id') + report['first_report'] = str(directory) + close(); process, driver, main_window = launch() + driver.click(wait(lambda: selected_cell(name))) + wait(lambda: driver.find(name+' · '+label, kind='Text'), timeout=60) + # Wait for startup recovery and at least one normal workspace polling interval. + time.sleep(2) + require(sql('SELECT attempt_id,input_version,generation,state,result_json FROM verification_attempts ORDER BY attempt_id')==attempts, 'Restart changed attempts or started another run') + verify_report(scope, run, directory, 'restart-readback', selected_only=bool(failed_name)) + report['checks'].append({'check_id': 'C06:restart', 'state': 'passed'}) + window_image(main_window.handle).save(output/'03-restart.png') + old_ids = {r['run_id'] for r in sql('SELECT run_id FROM verification_runs')} + driver.click(driver.find('重校验', kind='Button')) + rerun_id = wait(lambda: next((r['run_id'] for r in sql('SELECT run_id FROM verification_runs') if r['run_id'] not in old_ids), None)) + wait(lambda: finished(rerun_id), timeout=900) + rerun = run_record(rerun_id) + subset = dict(scope, items=[r for r in scope['items'] if r['display_name']==name]) + member = rerun['members'][0] + previous = next(r for r in attempts if r['attempt_id']==next(m['attempt_id'] for m in run['members'] if m['item_id']==member['item_id'])) + latest = sql('SELECT * FROM verification_attempts WHERE attempt_id=?', (member['attempt_id'],))[0] + require(latest['generation']==previous['generation']+1 and latest['input_version']==previous['input_version'], 'Rerun did not create the next attempt on the same input') + require(all(_hash(Path(p))==sha for p, sha in first_report_hashes.items()), 'Rerun overwrote the first report') + require(sql('SELECT result_json FROM verification_attempts WHERE attempt_id=?', (previous['attempt_id'],))[0]['result_json']==previous['result_json'], 'Rerun changed the old result') + second_report = export_report() + verify_report(subset, rerun, second_report, 'rerun', selected_only=True) + report['checks'].append({'check_id': 'C07:rerun', 'state': 'passed'}) + report['rerun_report'] = str(second_report) + require(all(_hash(Path(p))==sha for p, sha in original.items()), 'Input files changed during the loop') + require(sql('SELECT COUNT(*) total FROM execution_operations')[0]['total']==0, 'Replacement operations occurred') + close(); report['passed'] = True + except Exception as error: + report.update(passed=False, reason=str(error), error=type(error).__name__) + if driver: + for n, window in enumerate(driver.windows()): + try: window_image(window.handle).save(output/f'failure-{n}.png') + except Exception: pass + if process and process.poll() is None and main_window: + try: close() + except Exception: report['process_left_running'] = process.pid + elif process and process.poll() is None: + report['process_left_running'] = process.pid + finally: + report['input_sha256'] = original + (output/'acceptance.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8') + print(json.dumps(report, ensure_ascii=False)) + return 0 if report.get('passed') else 1 + + +if __name__=='__main__': raise SystemExit(main()) diff --git a/tools/verification_closed_loop.py b/tools/verification_closed_loop.py new file mode 100644 index 0000000..7dada64 --- /dev/null +++ b/tools/verification_closed_loop.py @@ -0,0 +1,144 @@ +"""Independent acceptance checks against a scope recorded BEFORE execution.""" +from html.parser import HTMLParser +import json +from pathlib import Path + +from dx_productivity.trial_task_center.native_netlist import _hash + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def records(service, task_id): + result, cursor = [], None + while True: + page = service.read_task(task_id, cursor=cursor) + result.extend(service.read_item(row['item_id']) for row in page['items']) + cursor = page['cursor'] + if cursor is None: + return result + + +def freeze_inputs(service, task_id, *, expected_count): + rows = records(service, task_id) + require(type(expected_count) is int and expected_count>0 and len(rows)==expected_count, + '验收必须事先指定非空、准确的工程数量。') + sources, items = {}, [] + for row in rows: + require(row['input_state']=='ready', '开始前输入尚未就绪。') + payload = row['payload'] + for name in ('before_ref', 'after_ref'): + capture = service.artifacts.read(payload[name], 'snapshot') + for path, sha in capture['source_hashes'].items(): + require(path not in sources or sources[path]==sha, '同一源文件出现不同摘要。') + require(_hash(Path(path))==sha, '源文件已偏离导入时的版本。') + sources[path] = sha + items.append({'item_id': row['item_id'], 'input_version': row['input_version'], + 'input_digest': payload['payload_digest'], 'before_ref': payload['before_ref'], + 'after_ref': payload['after_ref'], 'policy': payload['check_plan'], 'display_name': row['display_name']}) + operations = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0] + return {'standard': 'verification-closed-loop-v1', 'task_id': task_id, 'items': items, + 'source_hashes': sources, 'replacement_operations_before': operations} + + +class _ReportHTML(HTMLParser): + def __init__(self, text): + super().__init__(); self.mains = []; self.diagrams = 0 + self.feed(text) + + def handle_starttag(self, tag, attrs): + if tag=='main': self.mains.append(dict(attrs)) + if tag=='svg': self.diagrams += 1 + + +def verify_closed_loop(service, task_id, scope, run, report, *, expected_verdicts, selected_only=False): + checks = [] + def check(name, action): + try: + action() + checks.append({'check_id': name, 'state': 'passed'}) + except (ValueError, OSError, KeyError, TypeError) as error: + checks.append({'check_id': name, 'state': 'failed', 'reason': str(error)}) + + frozen = {r['item_id']: r for r in scope['items']} + members = {r['item_id']: r for r in run['members']} + report = Path(report) + current = {r['item_id']: r for r in records(service, task_id)} + if selected_only: + current = {key: row for key, row in current.items() if key in frozen} + def membership(): + require(scope['task_id']==task_id and bool(frozen), '冻结任务为空或身份错误。') + require(len(frozen)==len(scope['items']) and len(members)==len(run['members']), '工程或运行成员重复。') + require(set(frozen)==set(members)==set(current)==set(expected_verdicts), '冻结范围、运行成员、当前工程或预期清单不一致。') + durable = service.repo.connection.execute('SELECT state,members_json FROM verification_runs WHERE run_id=?', (run['run_id'],)).fetchone() + require(durable is not None and durable['state']=='completed', '运行未完成。') + persisted = json.loads(durable['members_json']) + require(all(row in persisted for row in run['members']) if selected_only else persisted==run['members'], + '运行成员与持久记录不同。') + require(all(v in ('CONSISTENT', 'WARNING', 'DIFFERENT') for v in expected_verdicts.values()), + '正常完整闭环必须事先指定可核验的工程结论。') + check('C01-C02:scope', membership) + + saved = {} + def report_scope(): + value = json.loads((report/'task.json').read_text(encoding='utf-8')) + require(value['task_id']==task_id, '报告任务不符。') + identities = {r['item_id'] for r in value['items']} + require(len(identities)==len(value['items']) and (set(frozen)<=identities if selected_only else identities==set(frozen)), '报告工程范围不符。') + require((report/'index.html').is_file(), '缺少报告首页。') + saved.update((r['item_id'], r) for r in value['items']) + check('C04:report_scope', report_scope) + + for item, original in frozen.items(): + def item_result(item=item, original=original): + row, member = current[item], members[item] + require(row['input_version']==member['input_version']==original['input_version'], '输入版本不符。') + require(row['payload']['payload_digest']==original['input_digest'], '输入摘要变化。') + for key in ('before_ref', 'after_ref'): + require(row['payload'][key]==original[key], '冻结副本引用变化。') + attempt = row['attempt'] + require(attempt is not None and attempt['state']=='succeeded', '尝试未成功完成,停止/失败不能算闭环。') + require(attempt['run_id']==run['run_id'] and row['attempt_id']==member['attempt_id'], '尝试不属于本次运行。') + service.validate_attempt(attempt) + result = attempt['result']; evidence = result['evidence']; binding = evidence['binding'] + require(result['verdict']==expected_verdicts[item], '工程结论与预期不符。') + for key, expected in (('item_id', item), ('attempt_id', attempt['attempt_id']), + ('generation', attempt['generation']), ('input_version', original['input_version']), + ('input_digest', original['input_digest']), ('policy', original['policy'])): + require(result[key]==expected, '结果身份不符:'+key) + for key in ('attempt_id', 'input_version', 'input_digest'): + require(binding[key]==result[key], '图表证据绑定不符:'+key) + require(result['resource_state']=='released', '执行资源退出未确认。') + released = json.loads((service.data_root/'verification/attempts'/attempt['attempt_id']/'resource-released.json').read_text(encoding='utf-8')) + require(released['resource_state']=='released' and released['owner_id']==attempt['owner_id'], '资源退出证明不属于本次尝试。') + if result['verdict'] in ('CONSISTENT', 'WARNING'): + states = {c['check_id']: c['state'] for c in result['checks']} + require(all(states.get(key) in ('equal', 'expected_change', 'warning') + for key in original['policy']['required_checks']), '必查项缺失或未知。') + else: + require(any(c['state']=='error' for c in result['checks']), '异常结论缺少错误事实。') + published = saved[item] + require(published['input_version']==original['input_version'] and published['attempt_id']==attempt['attempt_id'], + '报告引用旧输入或旧尝试。') + require(published['attempt']['result']==result, 'JSON报告事实与当前结果不同。') + html = _ReportHTML((report/f'{item}-v{original["input_version"]}.html').read_text(encoding='utf-8')) + require(len(html.mains)==1, 'HTML报告缺少唯一绑定。') + for key, value in (('item-id', item), ('attempt-id', attempt['attempt_id']), ('generation', attempt['generation']), + ('input-version', original['input_version']), ('input-digest', original['input_digest']), + ('evidence-digest', evidence['digest'])): + require(html.mains[0].get('data-'+key)==str(value), 'HTML报告绑定不符:'+key) + require(not evidence['pins'] or html.diagrams>0, '有逐脚事实却没有报告接线图。') + check('C02-C04:'+item, item_result) + + def unchanged(): + require(bool(scope['source_hashes']), '没有冻结源文件摘要。') + require(all(_hash(Path(path))==sha for path, sha in scope['source_hashes'].items()), '源文件被改写。') + count = service.repo.connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0] + # Early v1 standalone fixtures recorded an implicit zero baseline. + require(count==scope.get('replacement_operations_before', 0), '校验期间出现了新的换料操作或历史记录变化。') + check('C05:read_only', unchanged) + return {'standard': scope['standard'], 'task_id': task_id, 'run_id': run['run_id'], + 'validation_scope': 'single-run:C01-C05', 'not_run': ['C06', 'C07', 'C08'], + 'passed': all(c['state']=='passed' for c in checks), 'checks': checks} diff --git a/tools/verify_independent_verification.py b/tools/verify_independent_verification.py index 6ca376b..ac54542 100644 --- a/tools/verify_independent_verification.py +++ b/tools/verify_independent_verification.py @@ -12,6 +12,7 @@ from dx_productivity.trial_task_center.store import TaskStore from dx_productivity.trial_task_center.verification.service import VerificationService from dx_productivity.trial_task_center import native_netlist,netlist_verification from dx_productivity.trial_task_center.batch_resources import write_record +from verification_closed_loop import freeze_inputs,verify_closed_loop def main(argv=None): @@ -20,6 +21,8 @@ def main(argv=None): group.add_argument('--fixture-root',type=Path) group.add_argument('--native-before',type=Path) parser.add_argument('--native-after',type=Path);parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--expect-verdict',choices=('CONSISTENT','WARNING','DIFFERENT'),default='CONSISTENT', + help='Expected conclusion fixed before running; defaults to CONSISTENT') args=parser.parse_args(argv) if args.native_before and not args.native_after:parser.error('--native-after is required') root=args.output.resolve() @@ -48,6 +51,9 @@ def main(argv=None): task=service.create_task('独立校验验收',request_id='acceptance-create')['task_id'] accepted=service.import_pair(task,before,after,request_id='acceptance-import',stop_event=Event()) if not accepted['accepted']:raise ValueError('No input accepted') + frozen=freeze_inputs(service,task,expected_count=1) + expectations={row['item_id']:args.expect_verdict for row in frozen['items']} + write_record(root/'frozen-scope.json',dict(frozen,expected_verdicts=expectations,evidence_kind=evidence_kind)) run=service.start(task,request_id='acceptance-run') deadline=time.monotonic()+900 while service.busy: @@ -55,13 +61,15 @@ def main(argv=None): time.sleep(.05) current=service.read_task(task);report=service.export_task(task,root/'reports',request_id='acceptance-report') unchanged=all(native_netlist._hash(Path(p))==sha for p,sha in initial.items()) + closed_loop=verify_closed_loop(service,task,frozen,run,report,expected_verdicts=expectations) + write_record(root/'closed-loop.json',closed_loop) summary={'evidence_kind':evidence_kind,'task_id':task,'run':run,'summary':current['summary'], 'input_sha256':initial,'inputs_unchanged':unchanged,'report':str(report), - 'policy':'verification-v1','replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]} + 'policy':'verification-v1','closed_loop':closed_loop, + 'replacement_operations':store._connection.execute('SELECT COUNT(*) FROM execution_operations').fetchone()[0]} write_record(root/'acceptance.json',summary);print(json.dumps(summary,ensure_ascii=False)) if not unchanged:return 4 - if current['summary']['different']:return 2 - if current['summary']['unavailable'] or current['summary']['waiting'] or current['summary']['running']:return 3 + if not closed_loop['passed']:return 3 return 0 except Exception as error: write_record(root/'failure.json',{'error':type(error).__name__,'reason':str(error)})