Files
dxdesigner-part-replacement/tools/verify_harness_samples_ui.mjs

146 lines
8.7 KiB
JavaScript

/** Human-like inputs to the real UI, followed by independent artifact checks. */
import assert from 'node:assert/strict';
import {readFile,writeFile,mkdir} from 'node:fs/promises';
import {createHash} from 'node:crypto';
import path from 'node:path';
import {connect,awaitRuntimeOrigin} from './dsh_cdp.mjs';
import {evidenceFor} from './harness_session_evidence.mjs';
import {skipQuestion} from './dsh_skip_question.mjs';
import {workspaceCandidates,verifyWorkspaces} from './acceptance_runtime_state.mjs';
const output=path.resolve(process.argv[2]);
const manifest=JSON.parse(await readFile(path.join(output,'manifest.json'),'utf8'));
const c=await connect(Number(process.argv[3]),{expectedOrigin:manifest.runtime_origin});
if(manifest.runtime_origin)await awaitRuntimeOrigin(c,manifest.runtime_origin);
let report={git_head:manifest.git_head,source_patch_sha256:manifest.source_patch_sha256,
seed:manifest.seed,generation:manifest.generation,coverage:manifest.coverage,runtime:manifest.runtime,
driver:manifest.driver_description??'Qt mouse navigation + CDP mouse/keyboard Input',real_model:true,real_cad:false,samples:[]};
if(manifest.resume_completed){
const previous=JSON.parse(await readFile(path.join(output,'report.json'),'utf8'));
report.samples=previous.samples.filter(row=>row.turn_end||row.prompt);
}
const persist=()=>writeFile(path.join(output,'report.json'),JSON.stringify(report,null,2));
const visible=selector=>`[...document.querySelectorAll(${JSON.stringify(selector)})].filter(e=>e.getBoundingClientRect().width>0)`;
const composer=`(${visible('[contenteditable="true"]')})[0]`;
const replies=`[...document.querySelectorAll('[data-chat-group-part="response"]')].map(e=>e.innerText).join(String.fromCharCode(10))`;
const stop=`(${visible('button[aria-label="停止生成"]')}).length>0`;
async function wait(expression,timeout=30000){
const end=Date.now()+timeout;
while(Date.now()<end){if(await c.evaluate(expression))return;await new Promise(resolve=>setTimeout(resolve,200));}
throw Error('UI timeout: '+expression);
}
async function send(prompt){
await c.click(composer);
await c.call('Input.insertText',{text:prompt});
await wait(`${composer}?.textContent.includes(${JSON.stringify(prompt)})`);
const button=`(${visible('button[aria-label="发送消息"]')})[0]`;
await wait(`!!${button}&&!${button}.disabled`);
await c.click(button);
await wait(`!${composer}?.textContent.includes(${JSON.stringify(prompt)})&&document.body.innerText.includes(${JSON.stringify(prompt)})`);
}
async function validate(sample){
const checks=[];
for(const [filename,expected] of Object.entries(sample.expected)){
try{
const text=await readFile(path.join(sample.directory,filename),'utf8');
const actual=typeof expected==='string'?text:JSON.parse(text.replace(/^\uFEFF/,''));
assert.deepEqual(actual,expected);
checks.push({file:filename,passed:true});
}catch(error){checks.push({file:filename,passed:false,error:error.message});}
}
for(const [filename,digest] of Object.entries(sample.input_hashes)){
try{
const actual=createHash('sha256').update(await readFile(path.join(sample.directory,filename))).digest('hex');
assert.equal(actual,digest);checks.push({original:filename,passed:true});
}catch(error){checks.push({original:filename,passed:false,error:error.message});}
}
return checks;
}
try{
await wait(`!!${composer}`,60000);
for(const sample of manifest.cases){
if(report.samples.some(row=>row.id===sample.id))continue;
const row={id:sample.id,status:'running',started_at:new Date().toISOString(),prompt:null};
report.samples.push(row);await persist();
const started=Date.now();
try{
const newChat=`(${visible('button[aria-label="新建会话"]')}).at(-1)`;
await wait(`!!${newChat}`);
const previousSessionIds=new Set(manifest.require_workspace_preflight
?JSON.parse(await readFile(path.join(output,'runtime-state.json'),'utf8')).sessions.map(session=>session.id):[]);
await c.click(newChat);
await wait(`!!${composer}&&(${replies}).length===0`);
if(manifest.require_workspace_preflight){
let state,candidates=[];
const until=Date.now()+15000;
while(Date.now()<until){
state=JSON.parse(await readFile(path.join(output,'runtime-state.json'),'utf8'));
candidates=workspaceCandidates(state,previousSessionIds,manifest.workspace);
if(candidates.length)break;
await new Promise(resolve=>setTimeout(resolve,200));
}
verifyWorkspaces(candidates,manifest.workspace);
row.workspace_preflight={...state,verified_session_ids:candidates.map(session=>session.id)};
assert.ok(candidates.every(session=>session.shell_preflight?.passed),'Actual official shell preflight failed: '+JSON.stringify(candidates.map(session=>session.shell_preflight)));
}
const prompt=`这是本地验收任务,工作目录:${sample.directory}。所有输入都在这个目录,生成文件也放在这个目录。请实际完成操作,保留原始输入文件。\n${sample.request}\n完成后简要说明实际结果和产物路径;不能完成时明确说明原因。`;
row.prompt=prompt;await persist();
await send(prompt);
// Durable turn events decide completion. Tool-only or delayed-rendering
// responses need not expose text/a visible stop button within 60 seconds.
const deadline=Date.now()+(manifest.case_budget_ms??(sample.goal?600000:240000));
let sessionFile;
while(Date.now()<deadline){
const state=await c.evaluate(`({stopped:!(${stop}),reply:${replies},text:document.body.innerText})`);
const decline=`(${visible('button')}).find(e=>e.textContent.trim()==='拒绝')`;
const approval=await c.evaluate(`!!(${visible('button')}).find(e=>e.textContent.trim()==='允许一次')`);
if(approval){
(row.confirmations??=[]).push({decision:'deny',reason:'No OS permission changes are part of these file tasks',text:state.text.slice(-7000)});
await c.screenshot(path.join(output,sample.id+'-approval-'+row.confirmations.length+'.png'));
await c.click(decline);
}
await skipQuestion(c,output,sample.id);
const evidence=await evidenceFor(path.join(manifest.data,'dsh/harness-home'),prompt,sessionFile);
sessionFile=evidence.filename;
if(evidence.complete){
row.reply=state.reply;row.session_file=sessionFile;row.turn_end=evidence.reason;
row.goal=evidence.goal;
row.usage_events=evidence.events.filter(event=>event.data?.usage).map(event=>({seq:event.seq,type:event.type,usage:event.data.usage}));
row.model_routes=evidence.events.filter(event=>event.type==='request/context').map(event=>event.data);
row.event_count=evidence.events.length;
row.tool_calls=evidence.events.filter(event=>event.type==='tool/call').map(event=>({name:event.data.name,arguments:event.data.arguments}));
break;
}
await new Promise(resolve=>setTimeout(resolve,600));
}
if(row.reply===undefined)throw Error('Model task did not finish within the acceptance observation window');
row.checks=await validate(sample);
row.status=row.checks.every(check=>check.passed)?'passed':'failed';
if(row.turn_end?.kind!=='completed')row.status='failed';
if(sample.goal&&manifest.require_host_goal!==false&&row.goal?.phase!=='complete')row.status='failed';
await c.screenshot(path.join(output,sample.id+'.png'));
await writeFile(path.join(output,sample.id+'-reply.txt'),row.reply);
}catch(error){
if(!row.prompt){
report.samples.pop();report.status='setup_failed';report.setup_error=error.message;
report.setup_evidence=row.workspace_preflight;await persist();throw error;
}
row.status='failed';row.error=error.message;
row.reply=await c.evaluate(replies).catch(()=>null);
await c.screenshot(path.join(output,sample.id+'-failure.png')).catch(()=>{});
// An observer failure must never advance into another sample and abort
// the current model turn. Preserve it separately from semantic failures.
row.status='observation_failed';report.status='observation_failed';
report.observation_error=error.message;await persist();throw error;
}
row.seconds=Math.round((Date.now()-started)/10)/100;
await persist();console.log(JSON.stringify({id:row.id,status:row.status,seconds:row.seconds,error:row.error}));
}
report.passed=report.samples.filter(row=>row.status==='passed').length;
report.failed=report.samples.length-report.passed;
report.status=report.failed?'failed':'passed';
process.exitCode=report.failed?1:0;
await persist();
console.log(JSON.stringify({status:report.status,passed:report.passed,failed:report.failed}));
}finally{c.close();}