Files
dxdesigner-part-replacement/tools/summarize_paired_samples.mjs
T

68 lines
5.1 KiB
JavaScript

/** Independent paired report aggregation; preserves every failure and original report. */
import {readFile,writeFile} from 'node:fs/promises';
import path from 'node:path';
import assert from 'node:assert/strict';
import {execFileSync} from 'node:child_process';
import {fileURLToPath} from 'node:url';
import {readSession} from './harness_session_evidence.mjs';
const output=path.resolve(process.argv[2]);
const tools=path.dirname(fileURLToPath(import.meta.url));
const manifests=Object.fromEntries(await Promise.all(['native','harness'].map(async arm=>
[arm,JSON.parse(await readFile(path.join(output,arm,'manifest.json'),'utf8'))])));
assert.equal(manifests.native.purpose,manifests.harness.purpose,'Paired manifest purposes differ');
assert.ok(['exploration','holdout'].includes(manifests.native.purpose),'Missing or unsupported manifest purpose');
const arms={};
for(const arm of ['native','harness']){
const manifest=manifests[arm];
if(!process.argv.includes('--summary-only')){
execFileSync(process.execPath,[path.join(tools,'extract_acceptance_execution.mjs'),path.join(output,arm)]);
execFileSync(manifest.command[0],['-X','utf8',path.join(tools,'score_acceptance_semantics.py'),path.join(output,arm)]);
}
const report=JSON.parse(await readFile(path.join(output,arm,'report.json'),'utf8'));
const semantic=JSON.parse(await readFile(path.join(output,arm,'semantic-score.json'),'utf8'));
const interventions=(await readFile(path.join(output,arm,'interventions.jsonl'),'utf8').catch(()=>''))
.split('\n').filter(Boolean).map(line=>JSON.parse(line));
const rows=[];
for(const row of report.samples){
const events=row.session_file?await readSession(row.session_file):[];
rows.push({id:row.id,status:row.status,seconds:row.seconds,error:row.error,
checks:row.checks,tool_calls:row.tool_calls?.length??0,confirmations:row.confirmations?.length??0,
model_routes:events.filter(e=>e.type==='request/context').map(e=>e.data),
usage:events.filter(e=>e.data?.usage).map(e=>e.data.usage),
goal:row.goal,turn_end:row.turn_end,session_file:row.session_file});
rows.at(-1).interventions=interventions.filter(event=>event.id===row.id);
rows.at(-1).semantic=semantic.samples.find(sample=>sample.id===row.id);
rows.at(-1).tokens=Object.fromEntries(['inputTokens','outputTokens','totalTokens'].map(key=>
[key,rows.at(-1).usage.reduce((sum,usage)=>sum+(usage[key]??0),0)]));
}
arms[arm]={manifest,rows,semantic};
}
assert.equal(arms.native.manifest.seed,arms.harness.manifest.seed);
for(const [i,a]of arms.native.manifest.cases.entries()){
const b=arms.harness.manifest.cases[i];
assert.equal(a.request,b.request);assert.deepEqual(a.input_hashes,b.input_hashes);assert.deepEqual(a.expected,b.expected);
}
const summary={seed:arms.native.manifest.seed,real_model:true,real_cad:false,purpose:manifests.native.purpose,
matched_inputs:true,case_budget_ms:arms.native.manifest.case_budget_ms,
source_files:arms.native.manifest.source_files,
measurement_limits:['Session assistant-message token receipts exclude unreported background/title requests',
'Shared host load varied; elapsed times are exploratory, not a controlled performance benchmark',
'Windows shell ACL denials are environmental failures and are preserved'],arms:{}};
summary.workspace_layouts=Object.fromEntries(['native','harness'].map(arm=>[arm,
path.relative(path.join(arms[arm].manifest.data,'dsh/harness-home'),arms[arm].manifest.workspace)]));
summary.workspace_layouts_matched=summary.workspace_layouts.native===summary.workspace_layouts.harness;
if(!summary.workspace_layouts_matched)summary.measurement_limits.push('Exploration workspace layouts differ; use a fresh matched-layout holdout for benchmark conclusions');
for(const [arm,{manifest,rows,semantic}]of Object.entries(arms))summary.arms[arm]={runtime:manifest.runtime,
semantic_passed:semantic.passed,semantic_failed:semantic.failed,oracle_revision:semantic.oracle_revision,oracle_sha256:semantic.oracle_sha256,
completed:rows.filter(r=>r.status!=='running').length,passed:rows.filter(r=>r.status==='passed').length,
failed:rows.filter(r=>r.status==='failed').length,seconds:rows.reduce((s,r)=>s+(r.seconds??0),0),
tool_calls:rows.reduce((s,r)=>s+r.tool_calls,0),confirmations:rows.reduce((s,r)=>s+r.confirmations,0),
question_interruptions:rows.reduce((s,r)=>s+r.interventions.length,0),
tokens:Object.fromEntries(['inputTokens','outputTokens','totalTokens'].map(key=>[key,rows.reduce((s,r)=>s+r.tokens[key],0)])),
coverage:manifest.coverage,samples:rows};
const routes=arm=>[...new Set(arms[arm].rows.flatMap(row=>row.model_routes.map(route=>JSON.stringify({provider:route.provider,model:route.model}))))].sort();
summary.model_routes={native:routes('native'),harness:routes('harness')};
summary.model_routes_matched=JSON.stringify(routes('native'))===JSON.stringify(routes('harness'));
await writeFile(path.join(output,'paired-summary.json'),JSON.stringify(summary,null,2));
console.log(JSON.stringify(Object.fromEntries(Object.entries(summary.arms).map(([k,v])=>[k,{passed:v.passed,failed:v.failed,seconds:v.seconds,tool_calls:v.tool_calls}]))));