Files
dxdesigner-part-replacement/tools/summarize_harness_samples.mjs
T

68 lines
4.9 KiB
JavaScript

/** Audit persisted native sessions and publish a compact human-readable result. */
import {readFile,writeFile,copyFile,mkdir} from 'node:fs/promises';
import path from 'node:path';
import {evidenceFor} from './harness_session_evidence.mjs';
const output=path.resolve(process.argv[2]??'outputs/harness-acceptance-20');
const manifest=JSON.parse(await readFile(path.join(output,'manifest.json'),'utf8'));
const report=JSON.parse(await readFile(path.join(output,'report.json'),'utf8'));
const audit={git_head:report.git_head,real_model:report.real_model,real_cad:false,model:null,samples:[]};
for(const row of report.samples){
const sample=manifest.cases.find(value=>value.id===row.id);
const native=await evidenceFor(path.join(manifest.data,'dsh/harness-home'),row.prompt,row.session_file);
const events=native.events??[];
const results=events.filter(event=>event.type==='tool/result');
const failed=results.filter(event=>event.data.message?.isError||event.data.message?.content?.some(part=>
part.type==='text'&&(/\[exit code: [1-9]\d*\]/.test(part.text)||/\[sandbox: file access denied/.test(part.text))));
const usages=events.filter(event=>event.type==='assistant/message'&&event.data.usage).map(event=>event.data.usage);
const model=events.findLast(event=>event.type==='assistant/message')?.data.message.source;
audit.model??=model?.model;
const entry={id:row.id,status:row.status,seconds:row.seconds,session_id:events[0]?.id,
native_turn_completed:native.complete,goal:native.goal??null,tool_calls:events.filter(event=>event.type==='tool/call').length,
failed_tool_calls:failed.length,approval_interactions:row.confirmations?.length??0,
input_tokens:usages.reduce((total,usage)=>total+(usage.inputTokens??0),0),
output_tokens:usages.reduce((total,usage)=>total+(usage.outputTokens??0),0),
checks:row.checks??[],error:row.error??null};
if(row.id==='02-unicode'&&row.status==='passed'){
const source=await readFile(path.join(sample.directory,'中文目录/说明.txt'));
const copied=await readFile(path.join(sample.directory,'result.txt'));
entry.byte_copy_verified=source.equals(copied);
if(!entry.byte_copy_verified)entry.status='failed';
}
if(row.id==='20-repair-and-run'){
const calls=new Map(events.filter(event=>event.type==='tool/call').map(event=>[event.data.callId,event.data]));
entry.execution_verified=results.some(event=>{
const call=calls.get(event.data.message?.toolCallId);
const text=event.data.message?.content?.filter(part=>part.type==='text').map(part=>part.text).join('\n')??'';
return call?.name==='pwsh'&&call.arguments.includes('fixed.py')&&!/\[exit code: [1-9]\d*\]/.test(text)
&&/"sum"\s*:\s*12/.test(text)&&/"count"\s*:\s*3/.test(text);
});
if(row.status==='passed'&&!entry.execution_verified)entry.status='failed';
}
audit.samples.push(entry);
const destination=path.join(output,'evidence',row.id);
await mkdir(destination,{recursive:true});
if(native.filename)await copyFile(native.filename,path.join(destination,'native-session.jsonl.zstd'));
for(const filename of Object.keys(sample.expected)){
const target=path.join(destination,'artifacts',filename);
await mkdir(path.dirname(target),{recursive:true});
await copyFile(path.join(sample.directory,filename),target).catch(()=>{});
}
}
audit.completed=audit.samples.filter(row=>row.status!=='running').length;
audit.passed=audit.samples.filter(row=>row.status==='passed').length;
audit.failed=audit.samples.filter(row=>row.status==='failed').length;
audit.total=manifest.cases.length;
await writeFile(path.join(output,'audit.json'),JSON.stringify(audit,null,2));
const lines=[`# Harness 本地点击验收`, '',
`代码基线:${audit.git_head}。模型:${audit.model??'unknown'}。真实模型、真实界面;未执行真实 CAD。`, '',
`已结束 ${audit.completed}/${audit.total},产物检查通过 ${audit.passed},未通过 ${audit.failed}。`, '',
'权限确认中的操作系统权限修复不属于样本任务,验收者选择拒绝,记录交互次数;模型可继续使用已可用的文件能力完成目标。', '',
'| 样本 | 产物检查 | 原生回合结束 | 耗时秒 | 工具调用 | 失败调用 | 权限交互 |',
'| --- | --- | --- | ---: | ---: | ---: | ---: |',
...audit.samples.map(row=>`| ${row.id} | ${row.status} | ${row.native_turn_completed} | ${row.seconds??''} | ${row.tool_calls} | ${row.failed_tool_calls} | ${row.approval_interactions} |`), '',
'“产物检查通过”表示磁盘产物与预设答案一致,原文件哈希保留;不等于已经完成人工回复质量评分,也不代表优于旧版本。',
'各样本截图、回复、产物和原生压缩会话均保留在本目录。'];
await writeFile(path.join(output,'验收报告.md'),lines.join('\n'));
console.log(JSON.stringify({completed:audit.completed,total:audit.total,passed:audit.passed,failed:audit.failed,model:audit.model}));