/** Human-like inputs to the real UI, followed by independent artifact checks. */ import assert from 'node:assert/strict'; import {readFile,writeFile,mkdir} from 'node:fs/promises'; import {createHash} from 'node:crypto'; import path from 'node:path'; import {connect,awaitRuntimeOrigin} from './dsh_cdp.mjs'; import {evidenceFor} from './harness_session_evidence.mjs'; import {skipQuestion} from './dsh_skip_question.mjs'; import {workspaceCandidates,verifyWorkspaces} from './acceptance_runtime_state.mjs'; const output=path.resolve(process.argv[2]); const manifest=JSON.parse(await readFile(path.join(output,'manifest.json'),'utf8')); const c=await connect(Number(process.argv[3]),{expectedOrigin:manifest.runtime_origin}); if(manifest.runtime_origin)await awaitRuntimeOrigin(c,manifest.runtime_origin); let report={git_head:manifest.git_head,source_patch_sha256:manifest.source_patch_sha256, seed:manifest.seed,generation:manifest.generation,coverage:manifest.coverage,runtime:manifest.runtime, driver:manifest.driver_description??'Qt mouse navigation + CDP mouse/keyboard Input',real_model:true,real_cad:false,samples:[]}; if(manifest.resume_completed){ const previous=JSON.parse(await readFile(path.join(output,'report.json'),'utf8')); report.samples=previous.samples.filter(row=>row.turn_end||row.prompt); } const persist=()=>writeFile(path.join(output,'report.json'),JSON.stringify(report,null,2)); const visible=selector=>`[...document.querySelectorAll(${JSON.stringify(selector)})].filter(e=>e.getBoundingClientRect().width>0)`; const composer=`(${visible('[contenteditable="true"]')})[0]`; const replies=`[...document.querySelectorAll('[data-chat-group-part="response"]')].map(e=>e.innerText).join(String.fromCharCode(10))`; const stop=`(${visible('button[aria-label="停止生成"]')}).length>0`; async function wait(expression,timeout=30000){ const end=Date.now()+timeout; while(Date.now()setTimeout(resolve,200));} throw Error('UI timeout: '+expression); } async function send(prompt){ await c.click(composer); await c.call('Input.insertText',{text:prompt}); await wait(`${composer}?.textContent.includes(${JSON.stringify(prompt)})`); const button=`(${visible('button[aria-label="发送消息"]')})[0]`; await wait(`!!${button}&&!${button}.disabled`); await c.click(button); await wait(`!${composer}?.textContent.includes(${JSON.stringify(prompt)})&&document.body.innerText.includes(${JSON.stringify(prompt)})`); } async function validate(sample){ const checks=[]; for(const [filename,expected] of Object.entries(sample.expected)){ try{ const text=await readFile(path.join(sample.directory,filename),'utf8'); const actual=typeof expected==='string'?text:JSON.parse(text.replace(/^\uFEFF/,'')); assert.deepEqual(actual,expected); checks.push({file:filename,passed:true}); }catch(error){checks.push({file:filename,passed:false,error:error.message});} } for(const [filename,digest] of Object.entries(sample.input_hashes)){ try{ const actual=createHash('sha256').update(await readFile(path.join(sample.directory,filename))).digest('hex'); assert.equal(actual,digest);checks.push({original:filename,passed:true}); }catch(error){checks.push({original:filename,passed:false,error:error.message});} } return checks; } try{ await wait(`!!${composer}`,60000); for(const sample of manifest.cases){ if(report.samples.some(row=>row.id===sample.id))continue; const row={id:sample.id,status:'running',started_at:new Date().toISOString(),prompt:null}; report.samples.push(row);await persist(); const started=Date.now(); try{ const newChat=`(${visible('button[aria-label="新建会话"]')}).at(-1)`; await wait(`!!${newChat}`); const previousSessionIds=new Set(manifest.require_workspace_preflight ?JSON.parse(await readFile(path.join(output,'runtime-state.json'),'utf8')).sessions.map(session=>session.id):[]); await c.click(newChat); await wait(`!!${composer}&&(${replies}).length===0`); if(manifest.require_workspace_preflight){ let state,candidates=[]; const until=Date.now()+15000; while(Date.now()setTimeout(resolve,200)); } verifyWorkspaces(candidates,manifest.workspace); row.workspace_preflight={...state,verified_session_ids:candidates.map(session=>session.id)}; assert.ok(candidates.every(session=>session.shell_preflight?.passed),'Actual official shell preflight failed: '+JSON.stringify(candidates.map(session=>session.shell_preflight))); } const prompt=`这是本地验收任务,工作目录:${sample.directory}。所有输入都在这个目录,生成文件也放在这个目录。请实际完成操作,保留原始输入文件。\n${sample.request}\n完成后简要说明实际结果和产物路径;不能完成时明确说明原因。`; row.prompt=prompt;await persist(); await send(prompt); // Durable turn events decide completion. Tool-only or delayed-rendering // responses need not expose text/a visible stop button within 60 seconds. const deadline=Date.now()+(manifest.case_budget_ms??(sample.goal?600000:240000)); let sessionFile; while(Date.now()e.textContent.trim()==='拒绝')`; const approval=await c.evaluate(`!!(${visible('button')}).find(e=>e.textContent.trim()==='允许一次')`); if(approval){ (row.confirmations??=[]).push({decision:'deny',reason:'No OS permission changes are part of these file tasks',text:state.text.slice(-7000)}); await c.screenshot(path.join(output,sample.id+'-approval-'+row.confirmations.length+'.png')); await c.click(decline); } await skipQuestion(c,output,sample.id); const evidence=await evidenceFor(path.join(manifest.data,'dsh/harness-home'),prompt,sessionFile); sessionFile=evidence.filename; if(evidence.complete){ row.reply=state.reply;row.session_file=sessionFile;row.turn_end=evidence.reason; row.goal=evidence.goal; row.usage_events=evidence.events.filter(event=>event.data?.usage).map(event=>({seq:event.seq,type:event.type,usage:event.data.usage})); row.model_routes=evidence.events.filter(event=>event.type==='request/context').map(event=>event.data); row.event_count=evidence.events.length; row.tool_calls=evidence.events.filter(event=>event.type==='tool/call').map(event=>({name:event.data.name,arguments:event.data.arguments})); break; } await new Promise(resolve=>setTimeout(resolve,600)); } if(row.reply===undefined)throw Error('Model task did not finish within the acceptance observation window'); row.checks=await validate(sample); row.status=row.checks.every(check=>check.passed)?'passed':'failed'; if(row.turn_end?.kind!=='completed')row.status='failed'; if(sample.goal&&manifest.require_host_goal!==false&&row.goal?.phase!=='complete')row.status='failed'; await c.screenshot(path.join(output,sample.id+'.png')); await writeFile(path.join(output,sample.id+'-reply.txt'),row.reply); }catch(error){ if(!row.prompt){ report.samples.pop();report.status='setup_failed';report.setup_error=error.message; report.setup_evidence=row.workspace_preflight;await persist();throw error; } row.status='failed';row.error=error.message; row.reply=await c.evaluate(replies).catch(()=>null); await c.screenshot(path.join(output,sample.id+'-failure.png')).catch(()=>{}); // An observer failure must never advance into another sample and abort // the current model turn. Preserve it separately from semantic failures. row.status='observation_failed';report.status='observation_failed'; report.observation_error=error.message;await persist();throw error; } row.seconds=Math.round((Date.now()-started)/10)/100; await persist();console.log(JSON.stringify({id:row.id,status:row.status,seconds:row.seconds,error:row.error})); } report.passed=report.samples.filter(row=>row.status==='passed').length; report.failed=report.samples.length-report.passed; report.status=report.failed?'failed':'passed'; process.exitCode=report.failed?1:0; await persist(); console.log(JSON.stringify({status:report.status,passed:report.passed,failed:report.failed})); }finally{c.close();}