146 lines
8.7 KiB
JavaScript
146 lines
8.7 KiB
JavaScript
/** Human-like inputs to the real UI, followed by independent artifact checks. */
|
|
import assert from 'node:assert/strict';
|
|
import {readFile,writeFile,mkdir} from 'node:fs/promises';
|
|
import {createHash} from 'node:crypto';
|
|
import path from 'node:path';
|
|
import {connect,awaitRuntimeOrigin} from './dsh_cdp.mjs';
|
|
import {evidenceFor} from './harness_session_evidence.mjs';
|
|
import {skipQuestion} from './dsh_skip_question.mjs';
|
|
import {workspaceCandidates,verifyWorkspaces} from './acceptance_runtime_state.mjs';
|
|
|
|
const output=path.resolve(process.argv[2]);
|
|
const manifest=JSON.parse(await readFile(path.join(output,'manifest.json'),'utf8'));
|
|
const c=await connect(Number(process.argv[3]),{expectedOrigin:manifest.runtime_origin});
|
|
if(manifest.runtime_origin)await awaitRuntimeOrigin(c,manifest.runtime_origin);
|
|
let report={git_head:manifest.git_head,source_patch_sha256:manifest.source_patch_sha256,
|
|
seed:manifest.seed,generation:manifest.generation,coverage:manifest.coverage,runtime:manifest.runtime,
|
|
driver:manifest.driver_description??'Qt mouse navigation + CDP mouse/keyboard Input',real_model:true,real_cad:false,samples:[]};
|
|
if(manifest.resume_completed){
|
|
const previous=JSON.parse(await readFile(path.join(output,'report.json'),'utf8'));
|
|
report.samples=previous.samples.filter(row=>row.turn_end||row.prompt);
|
|
}
|
|
const persist=()=>writeFile(path.join(output,'report.json'),JSON.stringify(report,null,2));
|
|
const visible=selector=>`[...document.querySelectorAll(${JSON.stringify(selector)})].filter(e=>e.getBoundingClientRect().width>0)`;
|
|
const composer=`(${visible('[contenteditable="true"]')})[0]`;
|
|
const replies=`[...document.querySelectorAll('[data-chat-group-part="response"]')].map(e=>e.innerText).join(String.fromCharCode(10))`;
|
|
const stop=`(${visible('button[aria-label="停止生成"]')}).length>0`;
|
|
async function wait(expression,timeout=30000){
|
|
const end=Date.now()+timeout;
|
|
while(Date.now()<end){if(await c.evaluate(expression))return;await new Promise(resolve=>setTimeout(resolve,200));}
|
|
throw Error('UI timeout: '+expression);
|
|
}
|
|
async function send(prompt){
|
|
await c.click(composer);
|
|
await c.call('Input.insertText',{text:prompt});
|
|
await wait(`${composer}?.textContent.includes(${JSON.stringify(prompt)})`);
|
|
const button=`(${visible('button[aria-label="发送消息"]')})[0]`;
|
|
await wait(`!!${button}&&!${button}.disabled`);
|
|
await c.click(button);
|
|
await wait(`!${composer}?.textContent.includes(${JSON.stringify(prompt)})&&document.body.innerText.includes(${JSON.stringify(prompt)})`);
|
|
}
|
|
async function validate(sample){
|
|
const checks=[];
|
|
for(const [filename,expected] of Object.entries(sample.expected)){
|
|
try{
|
|
const text=await readFile(path.join(sample.directory,filename),'utf8');
|
|
const actual=typeof expected==='string'?text:JSON.parse(text.replace(/^\uFEFF/,''));
|
|
assert.deepEqual(actual,expected);
|
|
checks.push({file:filename,passed:true});
|
|
}catch(error){checks.push({file:filename,passed:false,error:error.message});}
|
|
}
|
|
for(const [filename,digest] of Object.entries(sample.input_hashes)){
|
|
try{
|
|
const actual=createHash('sha256').update(await readFile(path.join(sample.directory,filename))).digest('hex');
|
|
assert.equal(actual,digest);checks.push({original:filename,passed:true});
|
|
}catch(error){checks.push({original:filename,passed:false,error:error.message});}
|
|
}
|
|
return checks;
|
|
}
|
|
try{
|
|
await wait(`!!${composer}`,60000);
|
|
for(const sample of manifest.cases){
|
|
if(report.samples.some(row=>row.id===sample.id))continue;
|
|
const row={id:sample.id,status:'running',started_at:new Date().toISOString(),prompt:null};
|
|
report.samples.push(row);await persist();
|
|
const started=Date.now();
|
|
try{
|
|
const newChat=`(${visible('button[aria-label="新建会话"]')}).at(-1)`;
|
|
await wait(`!!${newChat}`);
|
|
const previousSessionIds=new Set(manifest.require_workspace_preflight
|
|
?JSON.parse(await readFile(path.join(output,'runtime-state.json'),'utf8')).sessions.map(session=>session.id):[]);
|
|
await c.click(newChat);
|
|
await wait(`!!${composer}&&(${replies}).length===0`);
|
|
if(manifest.require_workspace_preflight){
|
|
let state,candidates=[];
|
|
const until=Date.now()+15000;
|
|
while(Date.now()<until){
|
|
state=JSON.parse(await readFile(path.join(output,'runtime-state.json'),'utf8'));
|
|
candidates=workspaceCandidates(state,previousSessionIds,manifest.workspace);
|
|
if(candidates.length)break;
|
|
await new Promise(resolve=>setTimeout(resolve,200));
|
|
}
|
|
verifyWorkspaces(candidates,manifest.workspace);
|
|
row.workspace_preflight={...state,verified_session_ids:candidates.map(session=>session.id)};
|
|
assert.ok(candidates.every(session=>session.shell_preflight?.passed),'Actual official shell preflight failed: '+JSON.stringify(candidates.map(session=>session.shell_preflight)));
|
|
}
|
|
const prompt=`这是本地验收任务,工作目录:${sample.directory}。所有输入都在这个目录,生成文件也放在这个目录。请实际完成操作,保留原始输入文件。\n${sample.request}\n完成后简要说明实际结果和产物路径;不能完成时明确说明原因。`;
|
|
row.prompt=prompt;await persist();
|
|
await send(prompt);
|
|
// Durable turn events decide completion. Tool-only or delayed-rendering
|
|
// responses need not expose text/a visible stop button within 60 seconds.
|
|
const deadline=Date.now()+(manifest.case_budget_ms??(sample.goal?600000:240000));
|
|
let sessionFile;
|
|
while(Date.now()<deadline){
|
|
const state=await c.evaluate(`({stopped:!(${stop}),reply:${replies},text:document.body.innerText})`);
|
|
const decline=`(${visible('button')}).find(e=>e.textContent.trim()==='拒绝')`;
|
|
const approval=await c.evaluate(`!!(${visible('button')}).find(e=>e.textContent.trim()==='允许一次')`);
|
|
if(approval){
|
|
(row.confirmations??=[]).push({decision:'deny',reason:'No OS permission changes are part of these file tasks',text:state.text.slice(-7000)});
|
|
await c.screenshot(path.join(output,sample.id+'-approval-'+row.confirmations.length+'.png'));
|
|
await c.click(decline);
|
|
}
|
|
await skipQuestion(c,output,sample.id);
|
|
const evidence=await evidenceFor(path.join(manifest.data,'dsh/harness-home'),prompt,sessionFile);
|
|
sessionFile=evidence.filename;
|
|
if(evidence.complete){
|
|
row.reply=state.reply;row.session_file=sessionFile;row.turn_end=evidence.reason;
|
|
row.goal=evidence.goal;
|
|
row.usage_events=evidence.events.filter(event=>event.data?.usage).map(event=>({seq:event.seq,type:event.type,usage:event.data.usage}));
|
|
row.model_routes=evidence.events.filter(event=>event.type==='request/context').map(event=>event.data);
|
|
row.event_count=evidence.events.length;
|
|
row.tool_calls=evidence.events.filter(event=>event.type==='tool/call').map(event=>({name:event.data.name,arguments:event.data.arguments}));
|
|
break;
|
|
}
|
|
await new Promise(resolve=>setTimeout(resolve,600));
|
|
}
|
|
if(row.reply===undefined)throw Error('Model task did not finish within the acceptance observation window');
|
|
row.checks=await validate(sample);
|
|
row.status=row.checks.every(check=>check.passed)?'passed':'failed';
|
|
if(row.turn_end?.kind!=='completed')row.status='failed';
|
|
if(sample.goal&&manifest.require_host_goal!==false&&row.goal?.phase!=='complete')row.status='failed';
|
|
await c.screenshot(path.join(output,sample.id+'.png'));
|
|
await writeFile(path.join(output,sample.id+'-reply.txt'),row.reply);
|
|
}catch(error){
|
|
if(!row.prompt){
|
|
report.samples.pop();report.status='setup_failed';report.setup_error=error.message;
|
|
report.setup_evidence=row.workspace_preflight;await persist();throw error;
|
|
}
|
|
row.status='failed';row.error=error.message;
|
|
row.reply=await c.evaluate(replies).catch(()=>null);
|
|
await c.screenshot(path.join(output,sample.id+'-failure.png')).catch(()=>{});
|
|
// An observer failure must never advance into another sample and abort
|
|
// the current model turn. Preserve it separately from semantic failures.
|
|
row.status='observation_failed';report.status='observation_failed';
|
|
report.observation_error=error.message;await persist();throw error;
|
|
}
|
|
row.seconds=Math.round((Date.now()-started)/10)/100;
|
|
await persist();console.log(JSON.stringify({id:row.id,status:row.status,seconds:row.seconds,error:row.error}));
|
|
}
|
|
report.passed=report.samples.filter(row=>row.status==='passed').length;
|
|
report.failed=report.samples.length-report.passed;
|
|
report.status=report.failed?'failed':'passed';
|
|
process.exitCode=report.failed?1:0;
|
|
await persist();
|
|
console.log(JSON.stringify({status:report.status,passed:report.passed,failed:report.failed}));
|
|
}finally{c.close();}
|