31 lines
1.8 KiB
JavaScript
31 lines
1.8 KiB
JavaScript
/** Extract shell invocation receipts independently of the model's final claims. */
|
|
import {readFile,writeFile} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import {readSession} from './harness_session_evidence.mjs';
|
|
export function executionReceipts(events){
|
|
const receipts=[];
|
|
for(const call of events.filter(event=>event.type==='tool/call'&&event.data.name==='pwsh')){
|
|
let args;try{args=JSON.parse(call.data.arguments);}catch{continue;}
|
|
const command=args.command??'';
|
|
if(!/\b(?:python[\d.]*|py)(?:\.exe)?\b[^\r\n]*fixed\.py/i.test(command))continue;
|
|
const result=events.find(event=>event.type==='tool/result'&&event.data.message?.toolCallId===call.data.callId);
|
|
const text=result?.data.message?.content?.filter(part=>part.type==='text').map(part=>part.text).join('\n')??'';
|
|
const job=events.find(event=>event.type==='job/settled'&&event.data.toolCall?.callId===call.data.callId);
|
|
const exit=job?.data.detail?.match(/exit code:\s*(-?\d+)/i)??text.match(/\[exit code:\s*(-?\d+)\]/i);
|
|
const values=[];
|
|
for(const match of text.split('[stderr]')[0].matchAll(/\{[^\r\n]*\}/g)){try{values.push(JSON.parse(match[0]));}catch{}}
|
|
receipts.push({call_id:call.data.callId,command,exit_code:exit?Number(exit[1]):null,
|
|
tool_error:result?.data.message?.isError??true,output_json:values});
|
|
}
|
|
return receipts;
|
|
}
|
|
if(process.argv[1]?.endsWith('extract_acceptance_execution.mjs')){
|
|
const output=path.resolve(process.argv[2]);
|
|
const report=JSON.parse(await readFile(path.join(output,'report.json'),'utf8'));
|
|
const samples=[];
|
|
for(const row of report.samples){
|
|
if(row.session_file)samples.push({id:row.id,receipts:executionReceipts(await readSession(row.session_file))});
|
|
}
|
|
await writeFile(path.join(output,'execution-evidence.json'),JSON.stringify({revision:'shell-receipt-v2-job-exit',samples},null,2));
|
|
}
|