202 lines
14 KiB
Python
202 lines
14 KiB
Python
"""Run seeded stratified real-model tasks through the embedded UI in isolation."""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
from acceptance_workspace import create_workspace_root
|
||
import time
|
||
import zipfile
|
||
|
||
REPO = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(REPO / 'src'))
|
||
|
||
|
||
def fixtures(workspace):
|
||
cases = []
|
||
|
||
def add(name, files, request, expected, *, goal=False):
|
||
directory = workspace / f'{len(cases)+1:02d}-{name}'
|
||
directory.mkdir(parents=True)
|
||
for filename, text in files.items():
|
||
target = directory / filename
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
target.write_text(text, encoding='utf-8')
|
||
cases.append({'id': directory.name, 'directory': str(directory), 'request': request,
|
||
'expected': expected, 'goal': goal,
|
||
'input_hashes': {str(p.relative_to(directory)): hashlib.sha256(p.read_bytes()).hexdigest()
|
||
for p in directory.rglob('*') if p.is_file()}})
|
||
return directory
|
||
|
||
add('facts', {'notes.md': '工程 Board-A;目标位号 R1、R3;新料号 NEW-73;不执行CAD。\n'},
|
||
'阅读 notes.md,生成 result.json:project、references(数组)、new_part_number。',
|
||
{'result.json': {'project': 'Board-A', 'references': ['R1', 'R3'], 'new_part_number': 'NEW-73'}})
|
||
directory = add('unicode', {'中文目录/说明.txt': '第一行:电阻 Ω\n第二行:中文路径 ✓\n'},
|
||
'将中文目录/说明.txt 原样复制成 result.txt,保留两行文字和换行。',
|
||
{'result.txt': '第一行:电阻 Ω\n第二行:中文路径 ✓\n'})
|
||
cases[-1]['expected']['result.txt'] = (directory / '中文目录/说明.txt').read_bytes().decode('utf-8')
|
||
add('bom-totals', {'bom.csv': 'part,quantity\nA,2\nB,3\nA,4\n'},
|
||
'汇总 bom.csv 中每个料号数量,生成 result.json,格式为 {"A":数量,"B":数量,"total":总数量}。',
|
||
{'result.json': {'A': 6, 'B': 3, 'total': 9}})
|
||
add('duplicates', {'bom.csv': 'project,refdes,part\nBoard-A,R1,A\nBoard-A,R2,B\nBoard-A,R1,C\nBoard-B,R1,D\n'},
|
||
'按 project+refdes 查找重复位号。result.json 的 duplicates 数组只列重复身份,每项包含 project、refdes。不同工程同位号不是重复。',
|
||
{'result.json': {'duplicates': [{'project': 'Board-A', 'refdes': 'R1'}]}})
|
||
add('comparison', {'before.csv': 'refdes,part\nR1,A\nR2,B\nR3,C\n', 'after.csv': 'refdes,part\nR1,A\nR2,X\nR3,C\n'},
|
||
'比较 before.csv 和 after.csv,result.json 的 changes 数组记录改变的 refdes、before、after。',
|
||
{'result.json': {'changes': [{'refdes': 'R2', 'before': 'B', 'after': 'X'}]}})
|
||
add('missing-fields', {'bom.csv': 'refdes,old,new\nR1,A,B\nR2,B,\nR3,,C\n'},
|
||
'检查 bom.csv 每行 refdes、old、new 必填。result.json 的 invalid_rows 写 Excel 风格行号(表头第1行),只列无效行。',
|
||
{'result.json': {'invalid_rows': [3, 4]}})
|
||
add('mapping-copy', {'source.json': '{"R1":"OLD-A","R2":"KEEP","R3":"OLD-A"}'},
|
||
'把 source.json 中 OLD-A 替换成 NEW-A,保存 result.json;保留其他值和源文件。',
|
||
{'result.json': {'R1': 'NEW-A', 'R2': 'KEEP', 'R3': 'NEW-A'}})
|
||
add('nested-edit', {'config.json': '{"name":"Board-A","options":{"enabled":false,"limit":3},"note":"保留我"}'},
|
||
'读取 config.json,生成 result.json,只把 options.enabled 改为 true、options.limit 改为 7。其他字段原样保留。',
|
||
{'result.json': {'name': 'Board-A', 'options': {'enabled': True, 'limit': 7}, 'note': '保留我'}})
|
||
directory = add('zip', {}, '解压资料.zip,读取内部说明文件。生成 result.json:marker 填说明中的验收码,count 填清单中的数量。',
|
||
{'result.json': {'marker': 'ZIP-7391', 'count': 12}})
|
||
with zipfile.ZipFile(directory / '资料.zip', 'w') as archive:
|
||
archive.writestr('nested/readme.txt', '验收码 ZIP-7391\n')
|
||
archive.writestr('nested/count.json', '{"count":12}')
|
||
cases[-1]['input_hashes']['资料.zip'] = hashlib.sha256((directory / '资料.zip').read_bytes()).hexdigest()
|
||
add('log-cause', {'run.log': '09:00 INFO start\n09:01 ERROR LIBRARY_NOT_FOUND missing central.lmc\n09:02 ERROR EXECUTION_ABORTED\n09:03 INFO saved=0\n'},
|
||
'根据 run.log 找出第一条错误及保存数量。result.json 写 root_error、saved、execution_succeeded(布尔值)。不要把随后中止当作最初原因。',
|
||
{'result.json': {'root_error': 'LIBRARY_NOT_FOUND', 'saved': 0, 'execution_succeeded': False}})
|
||
add('recursive-files', {'a/one.txt': 'accept_id=A11\n', 'b/two.txt': 'accept_id=B22\n', 'b/deep/three.txt': 'accept_id=C33\n', 'ignore.md': '无编号'},
|
||
'递归检查所有 .txt 文件,提取 accept_id,按字母排序。result.json 写 ids 数组。',
|
||
{'result.json': {'ids': ['A11', 'B22', 'C33']}})
|
||
add('decimal', {'prices.csv': 'part,qty,unit_price\nA,3,1.25\nB,2,4.50\nC,7,1.00\n'},
|
||
'计算 prices.csv 总金额,生成 result.json:total 用两位小数字符串,currency 填 CNY。',
|
||
{'result.json': {'total': '19.75', 'currency': 'CNY'}})
|
||
add('ambiguous-project', {'catalog.json': '[{"name":"Board-A","path":"X:/a.prj"},{"name":"Board-A","path":"Y:/a.prj"}]'},
|
||
'用户只有工程名 Board-A,目录 catalog.json 中有两个同名工程。不要任选一个。result.json 写 status="ambiguous"、candidate_count、selected_path=null。',
|
||
{'result.json': {'status': 'ambiguous', 'candidate_count': 2, 'selected_path': None}})
|
||
add('large-table', {'numbers.csv': 'id,value\n'+''.join(f'{i},{i}\n' for i in range(1, 601))},
|
||
'核对 numbers.csv 全部数据,result.json 写 count、sum、min、max。必须实际读取完整表格。',
|
||
{'result.json': {'count': 600, 'sum': 180300, 'min': 1, 'max': 600}})
|
||
add('value-policy', {'rows.json': '[{"refdes":"R1","mode":"inherit"},{"refdes":"R2","mode":"preserve"},{"refdes":"R3","mode":"custom","custom":"47k"}]'},
|
||
'任务默认 Value 策略是 part_label。解析 rows.json:inherit 使用默认,其余使用显式策略。result.json 用位号作键,对应实际策略名;custom 项写 custom:47k。',
|
||
{'result.json': {'R1': 'part_label', 'R2': 'preserve', 'R3': 'custom:47k'}})
|
||
add('merge-files', {'a.json': '["R3","R1","R2"]', 'b.json': '["R2","R4"]'},
|
||
'合并 a.json、b.json,去重并排序,result.json 写 references 数组与 count。',
|
||
{'result.json': {'references': ['R1', 'R2', 'R3', 'R4'], 'count': 4}})
|
||
add('missing-file', {}, '尝试读取不存在的 missing-report.txt。若不存在,result.json 写 status="missing"、content=null;不要编造内容。',
|
||
{'result.json': {'status': 'missing', 'content': None}})
|
||
add('invalid-json', {'broken.json': '{"refdes":"R1","part":"A",}'},
|
||
'broken.json 无法按标准 JSON 解析。修正语法另存 result.json,保留原数据,不改变原文件。',
|
||
{'result.json': {'refdes': 'R1', 'part': 'A'}})
|
||
add('goal-workflow', {f'inputs/part-{i}.json': json.dumps({'id': i, 'quantity': i*3}) for i in range(1, 7)},
|
||
'建立 goal 并持续完成这个任务:逐个检查 inputs 中六份 JSON,为每个输入生成 outputs/part-N.json,包含 id 和 double_quantity(quantity乘2)。最后生成 result.json,count=6、total_double_quantity 为全部乘2后的和。核对所有产物后完成 goal。不要只给计划。',
|
||
{'result.json': {'count': 6, 'total_double_quantity': 126},
|
||
**{f'outputs/part-{i}.json': {'id': i, 'double_quantity': i*6} for i in range(1, 7)}}, goal=True)
|
||
add('repair-and-run', {'calculate.py': 'import json\nvalues=[2,4,6]\nresult=sum(values)+1\nprint(json.dumps({"sum":result,"count":len(values)}))\n'},
|
||
'calculate.py 求和有一个错误。保留原文件,写出修复后的 fixed.py,实际运行它,并把运行的 JSON 输出保存 result.json。预期 sum=12、count=3。必须实际运行验证,最后报告结果。',
|
||
{'result.json': {'sum': 12, 'count': 3}})
|
||
assert len(cases) == 20
|
||
return cases
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--output', type=Path, default=REPO / 'outputs/harness-acceptance-20')
|
||
parser.add_argument('--port', type=int, default=9231)
|
||
parser.add_argument('--provider-config', type=Path, default=Path(os.environ['LOCALAPPDATA']) / 'DxProductivity/trial-task-center/dsh')
|
||
parser.add_argument('--prepare-only', action='store_true')
|
||
parser.add_argument('--seed', type=int, help='Replay a recorded seed; omit for system randomness')
|
||
parser.add_argument('--count', type=int, default=20)
|
||
parser.add_argument('--legacy-fixed', action='store_true', help='Historical fixed regression cases; not random acceptance')
|
||
parser.add_argument('--case', action='append', help='Run named cases only, retaining their original sample IDs')
|
||
parser.add_argument('--windows-shell', action='store_true', help='Use the installed Windows PowerShell rather than the Codex process PATH')
|
||
args = parser.parse_args()
|
||
output = args.output.resolve()
|
||
output.mkdir(parents=True, exist_ok=True)
|
||
run_parent = Path(os.environ['LOCALAPPDATA']) / 'DxProductivity/acceptance-workspaces'
|
||
run_parent.mkdir(parents=True, exist_ok=True)
|
||
run = create_workspace_root(run_parent, 'click20-')
|
||
data = run / 'data'
|
||
workspace = data / 'dsh/harness-home/workspace'
|
||
from random_acceptance_cases import generate
|
||
seed, cases = (None, fixtures(workspace)) if args.legacy_fixed else generate(workspace, args.seed, args.count)
|
||
if args.case:
|
||
cases = [case for case in cases if case['id'] in args.case]
|
||
if len(cases) != len(set(args.case)):
|
||
raise ValueError('Unknown sample ID')
|
||
manifest = {'git_head': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=REPO, text=True).strip(),
|
||
'source_patch_sha256': hashlib.sha256(subprocess.check_output(['git', 'diff', '--binary', 'HEAD'], cwd=REPO)).hexdigest(),
|
||
'data': str(data), 'workspace': str(workspace), 'cases': cases, 'real_model': not args.prepare_only, 'real_cad': False,
|
||
'windows_shell': args.windows_shell,
|
||
'seed': str(seed) if seed is not None else None, 'generation': 'legacy-fixed' if args.legacy_fixed else 'stratified-random-v1',
|
||
'command': [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]],
|
||
'generator_sha256': hashlib.sha256((REPO / 'tools/random_acceptance_cases.py').read_bytes()).hexdigest(),
|
||
'coverage': sorted({case.get('stratum', case['id']) for case in cases}),
|
||
'runtime': 'embedded-harness', 'comparison': 'not-run',
|
||
'unsupported_coverage': ['real CAD', 'in-flight tool cancellation', 'native runtime paired baseline']}
|
||
manifest_path = output / 'manifest.json'
|
||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
print('MANIFEST='+str(manifest_path), flush=True)
|
||
if args.prepare_only:
|
||
return 0
|
||
from accept_dsh_desktop import copy_provider_configuration
|
||
copy_provider_configuration(args.provider_config, data / 'dsh')
|
||
if args.windows_shell:
|
||
shell = Path(os.environ['SystemRoot']) / 'System32/WindowsPowerShell/v1.0/powershell.exe'
|
||
profile = data / 'dsh/harness-home/profiles/dx-task/cordis.patch.yml'
|
||
profile.parent.mkdir(parents=True, exist_ok=True)
|
||
profile.write_text(json.dumps([{'id': 'pwsh-sandbox', 'config': {'pwshPath': str(shell)}}]), encoding='utf-8')
|
||
(data / 'dsh/issue-config.json').write_text('{"enabled":false}', encoding='utf-8')
|
||
os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = f'127.0.0.1:{args.port}'
|
||
os.environ.pop('QT_QPA_PLATFORM', None)
|
||
from PySide6.QtCore import QTimer, Qt
|
||
from PySide6.QtTest import QTest
|
||
from PySide6.QtWidgets import QApplication, QMessageBox
|
||
from dx_productivity.trial_task_center import application
|
||
from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow
|
||
state = {'phase': 'open', 'started': time.monotonic(), 'child': None}
|
||
|
||
def build(service):
|
||
window = TrialTaskCenterWindow(service)
|
||
window.resize(1380, 900)
|
||
|
||
def tick():
|
||
modal = QApplication.activeModalWidget()
|
||
if isinstance(modal, QMessageBox) and modal.windowTitle().startswith('DSH助手'):
|
||
button = modal.button(QMessageBox.StandardButton.No)
|
||
if button:
|
||
QTest.mouseClick(button, Qt.MouseButton.LeftButton)
|
||
if state['phase'] == 'open':
|
||
QTest.mouseClick(window.dsh_navigation, Qt.MouseButton.LeftButton)
|
||
state['phase'] = 'starting'
|
||
elif state['phase'] == 'starting' and window._dsh_view is not None and window._dsh_view._started:
|
||
log = (output / 'driver.log').open('w', encoding='utf-8')
|
||
state['child'] = subprocess.Popen(['node', str(REPO / 'tools/verify_harness_samples_ui.mjs'),
|
||
str(output), str(args.port)], cwd=REPO, stdout=log, stderr=subprocess.STDOUT,
|
||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||
state['phase'] = 'running'
|
||
elif state['phase'] == 'running' and state['child'].poll() is not None:
|
||
window.grab().save(str(output / 'desktop-final.png'))
|
||
timer.stop()
|
||
state['phase'] = 'done'
|
||
window.close()
|
||
elif state['phase'] == 'starting' and time.monotonic()-state['started'] > 360:
|
||
(output / 'startup-error.txt').write_text(window._dsh_view.last_error if window._dsh_view else 'No view', encoding='utf-8')
|
||
timer.stop()
|
||
window.close()
|
||
|
||
timer = QTimer(window)
|
||
timer.setInterval(250)
|
||
timer.timeout.connect(tick)
|
||
timer.start()
|
||
return window
|
||
|
||
application._load_window = lambda: build
|
||
code = application.main(['--data-root', str(data)])
|
||
return code or (state['child'].returncode if state['child'] else 1)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
raise SystemExit(main())
|